From 09075b983330efe1a925ec1c8223601bf7cc104f Mon Sep 17 00:00:00 2001 From: gmega Date: Wed, 15 Nov 2023 14:26:06 -0300 Subject: [PATCH] fix log query when no start/end date is provided --- .../elastic_search/elastic_search_source.py | 15 +- ...etch_logs_when_no_dates_are_specified.yaml | 10176 ++++++++++++++++ .../tests/test_elasticsearch_source.py | 12 + logtools/log/tests/__init__.py | 0 logtools/log/tests/test_tree.py | 16 + logtools/log/utils.py | 9 + pyproject.toml | 2 +- 7 files changed, 10223 insertions(+), 7 deletions(-) create mode 100644 logtools/log/sources/input/elastic_search/tests/cassettes/test_elasticsearch_source/test_should_fetch_logs_when_no_dates_are_specified.yaml create mode 100644 logtools/log/tests/__init__.py create mode 100644 logtools/log/tests/test_tree.py create mode 100644 logtools/log/utils.py diff --git a/logtools/log/sources/input/elastic_search/elastic_search_source.py b/logtools/log/sources/input/elastic_search/elastic_search_source.py index 4aa0e96..5a89b58 100644 --- a/logtools/log/sources/input/elastic_search/elastic_search_source.py +++ b/logtools/log/sources/input/elastic_search/elastic_search_source.py @@ -6,6 +6,7 @@ from typing import Optional, Dict, Any, Iterator, Set from elasticsearch import Elasticsearch from logtools.log.base import TimestampedLogLine, LogSource +from logtools.log.utils import tree logger = logging.getLogger(__name__) @@ -48,7 +49,8 @@ class ElasticSearchSource(LogSource[TimestampedLogLine[ElasticSearchLocation]]): def _indices(self) -> Iterator[str]: # FIXME this is a VERY INEFFICIENT fallback if self.start_date is None: - return [f'{INDEX_PREFIX}-*'] + yield f'{INDEX_PREFIX}-*' + return start_day = self.start_date.date() end_day = self.end_date.date() if self.end_date else datetime.now().date() @@ -61,9 +63,8 @@ class ElasticSearchSource(LogSource[TimestampedLogLine[ElasticSearchLocation]]): start_day += increment def _build_query(self) -> Dict[str, Any]: - query: Dict[str, Any] = { - 'sort': [{'@timestamp': 'asc'}] - } + query = tree() + query['sort'] = [{'@timestamp': 'asc'}] if self.start_date is not None or self.end_date is not None: time_range = {} @@ -74,10 +75,12 @@ class ElasticSearchSource(LogSource[TimestampedLogLine[ElasticSearchLocation]]): query['query'] = {'bool': {'filter': [{'range': {'@timestamp': time_range}}]}} if self.pods is not None: - query['query']['bool']['filter'].append({"terms": {"pod_name.keyword": list(self.pods)}}) + filters = query['query']['bool'].setdefault('filter', []) + filters.append({"terms": {"pod_name.keyword": list(self.pods)}}) if self.run_id is not None: - query['query']['bool']['filter'].append({"term": {"pod_labels.runid.keyword": self.run_id}}) + filters = query['query']['bool'].setdefault('filter', []) + filters.append({"term": {"pod_labels.runid.keyword": self.run_id}}) if 'query' not in query: query['query'] = {'match_all': {}} diff --git a/logtools/log/sources/input/elastic_search/tests/cassettes/test_elasticsearch_source/test_should_fetch_logs_when_no_dates_are_specified.yaml b/logtools/log/sources/input/elastic_search/tests/cassettes/test_elasticsearch_source/test_should_fetch_logs_when_no_dates_are_specified.yaml new file mode 100644 index 0000000..b184bde --- /dev/null +++ b/logtools/log/sources/input/elastic_search/tests/cassettes/test_elasticsearch_source/test_should_fetch_logs_when_no_dates_are_specified.yaml @@ -0,0 +1,10176 @@ +interactions: +- request: + body: '{"query":{"bool":{"filter":[{"terms":{"pod_name.keyword":["codex1-3-b558568cf-tvcsc"]}},{"term":{"pod_labels.runid.keyword":"20231109-101554"}}]}},"size":5000,"sort":[{"@timestamp":"asc"}]}' + headers: + accept: + - application/vnd.elasticsearch+json; compatible-with=8 + connection: + - keep-alive + content-type: + - application/vnd.elasticsearch+json; compatible-with=8 + user-agent: + - elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0) + x-elastic-client-meta: + - es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7 + method: POST + uri: http://localhost:9200/continuous-tests-pods-*/_search?scroll=2m + response: + body: + string: '{"_scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDnF1ZXJ5VGhlbkZldGNoDhYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABna8WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABnbAWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABnbIWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABnbEWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABnbMWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABnbQWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABnbUWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABnbcWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABnbYWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABnbgWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABnbkWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABnboWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABnbsWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABnbwWVmdpQjdNT1RUM3VuSmxXekR6ZnREZw==","took":2194,"timed_out":false,"_shards":{"total":14,"successful":14,"skipped":0,"failed":0},"hits":{"total":{"value":11138278,"relation":"eq"},"max_score":null,"hits":[{"_index":"continuous-tests-pods-2023.11.09","_id":"PeQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.804677280Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"Set + CODEX_NAT: 10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583804]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PuQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.807091646Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"Private + key set","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583807]},{"_index":"continuous-tests-pods-2023.11.09","_id":"P-QutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.807116882Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"Run + Codex node","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583807]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QOQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.815001340Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"NOT + 2023-11-09 22:23:03.814+00:00 Starting metrics HTTP server tid=1 + url=http://0.0.0.0:8083/metrics count=1","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583815]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QeQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.815078251Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"WRN + 2023-11-09 22:23:03.814+00:00 Data directory has insecure permissions. Correcting + them. tid=1 data_dir=datadir3 current_permissions=0755 required_permissions=0700 + count=2","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583815]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QuQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.815202291Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.814+00:00 Data dir initialized topics=\"codex\" + tid=1 dir=datadir3 count=3","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583815]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Q-QutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.816374962Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.816+00:00 Repo dir initialized topics=\"codex\" + tid=1 dir=datadir3/repo count=4","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583816]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ROQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.816398928Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"INF + 2023-11-09 22:23:03.816+00:00 Creating a private key and saving it tid=1 + count=5","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583816]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ReQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.834517418Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.834+00:00 Updating announce record topics=\"codex + discovery\" tid=1 addrs=@[/ip4/0.0.0.0/tcp/8082] count=6","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583834]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RuQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.846371007Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.846+00:00 create everything but rest server topics=\"codex\" + tid=1 count=7","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583846]},{"_index":"continuous-tests-pods-2023.11.09","_id":"R-QutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.846744764Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"NOT + 2023-11-09 22:23:03.846+00:00 Starting codex node topics=\"codex + node\" tid=1 count=8","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583846]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SOQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.846768689Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.846+00:00 Starting repo topics=\"codex + repostore\" tid=1 count=9","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583846]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SeQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.847060431Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.846+00:00 Number of blocks in store at start topics=\"codex + repostore\" tid=1 total=0 count=10","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583847]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SuQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.847078176Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"NOT + 2023-11-09 22:23:03.846+00:00 Current bytes used for cache quota topics=\"codex + repostore\" tid=1 bytes=0 count=11","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583847]},{"_index":"continuous-tests-pods-2023.11.09","_id":"S-QutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.847096861Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"NOT + 2023-11-09 22:23:03.847+00:00 Current bytes used for persist quota topics=\"codex + repostore\" tid=1 bytes=0 count=12","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583847]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TOQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.847227690Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.847+00:00 Timer starting: tid=1 + name=\"Unnamed Timer\" count=13","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583847]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TeQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.848151518Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.847+00:00 Updating announce record topics=\"codex + discovery\" tid=1 addrs=@[/ip4/10.244.3.62/tcp/8082] count=14","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583848]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TuQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.848173739Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.847+00:00 Updating Dht record topics=\"codex + discovery\" tid=1 ip=10.244.3.62 port=8081 count=15","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583848]},{"_index":"continuous-tests-pods-2023.11.09","_id":"T-QutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.848880323Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.848+00:00 Discovery engine start topics=\"codex + discoveryengine\" tid=1 count=16","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583848]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UOQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.849255883Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.849+00:00 Begin iterating blocks... topics=\"codex + discoveryengine\" tid=1 count=17","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583849]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UeQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.849270958Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.849+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=18","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583849]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UuQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.849275308Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.849+00:00 Blockexc starting with concurrent tasks topics=\"codex + blockexcengine\" tid=1 tasks=10 count=19","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583849]},{"_index":"continuous-tests-pods-2023.11.09","_id":"U-QutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.849279495Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.849+00:00 Starting blockexc task runner topics=\"codex + blockexcengine\" tid=1 count=20","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583849]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VOQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.849283460Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.849+00:00 Starting blockexc task runner topics=\"codex + blockexcengine\" tid=1 count=21","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583849]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VeQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.849844515Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.849+00:00 Starting blockexc task runner topics=\"codex + blockexcengine\" tid=1 count=22","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583849]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VuQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.849862297Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.849+00:00 Starting blockexc task runner topics=\"codex + blockexcengine\" tid=1 count=23","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583849]},{"_index":"continuous-tests-pods-2023.11.09","_id":"V-QutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.849867240Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.849+00:00 Starting blockexc task runner topics=\"codex + blockexcengine\" tid=1 count=24","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583849]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WOQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.849947920Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.849+00:00 Starting blockexc task runner topics=\"codex + blockexcengine\" tid=1 count=25","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583849]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WeQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.849964797Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.849+00:00 Starting blockexc task runner topics=\"codex + blockexcengine\" tid=1 count=26","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583849]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WuQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.849969735Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.849+00:00 Starting blockexc task runner topics=\"codex + blockexcengine\" tid=1 count=27","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583849]},{"_index":"continuous-tests-pods-2023.11.09","_id":"W-QutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.850013106Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.849+00:00 Starting blockexc task runner topics=\"codex + blockexcengine\" tid=1 count=28","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583850]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XOQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.850029152Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.849+00:00 Starting blockexc task runner topics=\"codex + blockexcengine\" tid=1 count=29","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583850]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XeQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.850523337Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.850+00:00 Cleaning up expired records tid=1 + count=30","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583850]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XuQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.850767330Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.850+00:00 Cleaned up expired records tid=1 + size=0 count=31","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583850]},{"_index":"continuous-tests-pods-2023.11.09","_id":"X-QutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.850779989Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.850+00:00 Cleaning up query iterator tid=1 + count=32","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583850]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YOQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.850784090Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.850+00:00 Cleaning up orphaned records tid=1 + count=33","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583850]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YeQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.851131266Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.850+00:00 Cleaning up query iterator tid=1 + count=34","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583851]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YuQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.851560083Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.851+00:00 Iterating blocks finished. topics=\"codex + discoveryengine\" tid=1 count=35","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583851]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Y-QutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.851572158Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.851+00:00 About to sleep advertise loop topics=\"codex + discoveryengine\" tid=1 sleep=30m count=36","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583851]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZOQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.858337962Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 22:23:03.858+00:00 Fetched my slots topics=\"marketplace + onchain market\" tid=1 numSlots=0 count=37","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583858]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZeQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.858368019Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"INF + 2023-11-09 22:23:03.858+00:00 Loading active slots topics=\"sales + marketplace\" tid=1 slotsCount=0 count=38","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583858]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZuQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.858610677Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"INF + 2023-11-09 22:23:03.858+00:00 found unused reservations for deletion topics=\"sales + marketplace\" tid=1 unused=0 count=39","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583858]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Z-QutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.858633121Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.858+00:00 starting slot queue topics=\"marketplace + slotqueue\" tid=1 count=40","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583858]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aOQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.858767941Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.858+00:00 adding new worker to worker queue topics=\"marketplace + slotqueue\" tid=1 count=41","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583858]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aeQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.858780091Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.858+00:00 adding new worker to worker queue topics=\"marketplace + slotqueue\" tid=1 count=42","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583858]},{"_index":"continuous-tests-pods-2023.11.09","_id":"auQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.858800075Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:03.858+00:00 adding new worker to worker queue topics=\"marketplace + slotqueue\" tid=1 count=43","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583858]},{"_index":"continuous-tests-pods-2023.11.09","_id":"a-QutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.869714161Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"NOT + 2023-11-09 22:23:03.869+00:00 Started codex node topics=\"codex + node\" tid=1 id=16Uiu2HAmMembVit2otv2uipskTxaoNAW7pxQnoP37d5fByYcZYzP addrs=@[/ip4/0.0.0.0/tcp/8082] + count=44","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583869]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bOQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:03.869762748Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"NOT + 2023-11-09 22:23:03.869+00:00 REST service started tid=1 + address=0.0.0.0:8080 count=45","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568583869]},{"_index":"continuous-tests-pods-2023.11.09","_id":"beQutosBOvK4VyD1mZ1H","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:06.850761818Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:06.850+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=46","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568586850]},{"_index":"continuous-tests-pods-2023.11.09","_id":"n-QutosBOvK4VyD1sbpa","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:09.350437231Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 22:23:09.349+00:00 Received request topics=\"codex\" + tid=1 peer=10.244.0.212:48570 meth=GET uri=/api/codex/v1/debug/info count=47","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568589350]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oOQutosBOvK4VyD1sbpa","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:09.350493268Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 22:23:09.350+00:00 Serving API request topics=\"codex\" + tid=1 peer=10.244.0.212:48570 meth=GET uri=/api/codex/v1/debug/info path_params=\"(table: + {:})\" query_params=\"(table: {:})\" content_body=none(ContentBody) count=48","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568589350]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oeQutosBOvK4VyD1sbpa","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:09.351096825Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 22:23:09.350+00:00 Received response from handler topics=\"codex\" + tid=1 status=200 meth=GET peer=10.244.0.212:48570 uri=/api/codex/v1/debug/info + content_type=application/json content_size=2433 count=49","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568589351]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ouQutosBOvK4VyD1sbpa","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:09.852288498Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:09.851+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=50","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568589852]},{"_index":"continuous-tests-pods-2023.11.09","_id":"o-QutosBOvK4VyD1sbpa","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:12.853472647Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:12.853+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=51","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568592853]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pOQutosBOvK4VyD1sbpa","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:13.043566282Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 22:23:13.043+00:00 Received request topics=\"codex\" + tid=1 peer=10.244.0.212:49966 meth=GET uri=/api/codex/v1/debug/info count=52","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568593043]},{"_index":"continuous-tests-pods-2023.11.09","_id":"peQutosBOvK4VyD1sbpa","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:13.043638769Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 22:23:13.043+00:00 Serving API request topics=\"codex\" + tid=1 peer=10.244.0.212:49966 meth=GET uri=/api/codex/v1/debug/info path_params=\"(table: + {:})\" query_params=\"(table: {:})\" content_body=none(ContentBody) count=53","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568593043]},{"_index":"continuous-tests-pods-2023.11.09","_id":"puQutosBOvK4VyD1sbpa","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:13.043919373Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 22:23:13.043+00:00 Received response from handler topics=\"codex\" + tid=1 status=200 meth=GET peer=10.244.0.212:49966 uri=/api/codex/v1/debug/info + content_type=application/json content_size=2433 count=54","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568593043]},{"_index":"continuous-tests-pods-2023.11.09","_id":"p-QutosBOvK4VyD1sbpa","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:13.052128038Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 22:23:13.051+00:00 Received request topics=\"codex\" + tid=1 peer=10.244.0.212:49980 meth=POST uri=/api/codex/v1/sales/availability + count=55","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568593052]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qOQutosBOvK4VyD1sbpa","_score":null,"_ignored":["message.keyword"],"_source":{"@timestamp":"2023-11-09T22:23:13.052215448Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 22:23:13.051+00:00 Serving API request topics=\"codex\" + tid=1 peer=10.244.0.212:49980 meth=POST uri=/api/codex/v1/sales/availability + path_params=\"(table: {:})\" query_params=\"(table: {:})\" content_body=none(ContentBody) + count=56","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568593052]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qeQutosBOvK4VyD1sbpa","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:13.052636679Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:13.052+00:00 creating availability topics=\"sales + reservations\" tid=1 size=1073741824 duration=3600000 minPrice=1024 maxCollateral=1024 + count=57","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568593052]},{"_index":"continuous-tests-pods-2023.11.09","_id":"quQutosBOvK4VyD1sbpa","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:13.052716098Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:13.052+00:00 Reserving bytes topics=\"codex + repostore\" tid=1 reserved=0 bytes=1073741824 count=58","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568593052]},{"_index":"continuous-tests-pods-2023.11.09","_id":"q-QutosBOvK4VyD1sbpa","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:13.055422766Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:13.055+00:00 updating Availability topics=\"sales + reservations\" tid=1 id=2d1adfc7a9b2761844406bd9a1753c2877bcd9333beb4d07d431c91906a1c326 + size=1073741824 count=59","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568593055]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rOQutosBOvK4VyD1sbpa","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:13.057435588Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:13.057+00:00 availability added, querying past storage requests + to add to queue topics=\"marketplace sales onAvailabilityAdded callback\" + tid=1 count=60","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568593057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"reQutosBOvK4VyD1sbpa","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:13.061929525Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:13.061+00:00 no storage request events found in recent past + topics=\"marketplace sales onAvailabilityAdded callback\" tid=1 count=61","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568593061]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ruQutosBOvK4VyD1sbpa","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:13.061997092Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 22:23:13.061+00:00 Received response from handler topics=\"codex\" + tid=1 status=200 meth=POST peer=10.244.0.212:49980 uri=/api/codex/v1/sales/availability + content_type=application/json content_size=157 count=62","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568593061]},{"_index":"continuous-tests-pods-2023.11.09","_id":"n-QutosBOvK4VyD1yehw","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:15.855633667Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:15.855+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=63","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568595855]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oOQutosBOvK4VyD1yehw","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:18.857178912Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:18.856+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=64","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568598857]},{"_index":"continuous-tests-pods-2023.11.09","_id":"z-UutosBOvK4VyD14SuG","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:21.858737902Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:21.858+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=65","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568601858]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0OUutosBOvK4VyD14SuG","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:23.184794722Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 22:23:23.184+00:00 Received request topics=\"codex\" + tid=1 peer=10.244.0.212:38596 meth=GET uri=/api/codex/v1/debug/info count=66","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568603184]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0eUutosBOvK4VyD14SuG","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:23.184831793Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 22:23:23.184+00:00 Serving API request topics=\"codex\" + tid=1 peer=10.244.0.212:38596 meth=GET uri=/api/codex/v1/debug/info path_params=\"(table: + {:})\" query_params=\"(table: {:})\" content_body=none(ContentBody) count=67","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568603184]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0uUutosBOvK4VyD14SuG","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:23.185078327Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 22:23:23.184+00:00 Received response from handler topics=\"codex\" + tid=1 status=200 meth=GET peer=10.244.0.212:38596 uri=/api/codex/v1/debug/info + content_type=application/json content_size=2435 count=68","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568603185]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0-UutosBOvK4VyD14SuG","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:23.243403192Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 22:23:23.243+00:00 Received request topics=\"codex\" + tid=1 peer=10.244.0.212:38602 meth=GET uri=/api/codex/v1/debug/peer/16Uiu2HAkwmYFvu5wWJqvd85yPk3aSX4Q6rehifJLnxHbjDpYW64F + count=69","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568603243]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1OUutosBOvK4VyD14SuG","_score":null,"_ignored":["message.keyword"],"_source":{"@timestamp":"2023-11-09T22:23:23.243481760Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 22:23:23.243+00:00 Serving API request topics=\"codex\" + tid=1 peer=10.244.0.212:38602 meth=GET uri=/api/codex/v1/debug/peer/16Uiu2HAkwmYFvu5wWJqvd85yPk3aSX4Q6rehifJLnxHbjDpYW64F + path_params=\"(table: {\\\"peerid\\\": @[\\\"16Uiu2HAkwmYFvu5wWJqvd85yPk3aSX4Q6rehifJLnxHbjDpYW64F\\\"]})\" + query_params=\"(table: {:})\" content_body=none(ContentBody) count=70","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568603243]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1eUutosBOvK4VyD14SuG","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:23.243494110Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:23.243+00:00 debug/peer start topics=\"codex + restapi\" tid=1 count=71","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568603243]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1uUutosBOvK4VyD14SuG","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:23.243527585Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:23.243+00:00 protocol.resolve... topics=\"codex + discovery\" tid=1 count=72","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568603243]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1-UutosBOvK4VyD14SuG","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:23.245623996Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:23.245+00:00 debug/peer returning peer record topics=\"codex + restapi\" tid=1 count=73","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568603245]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2OUutosBOvK4VyD14SuG","_score":null,"_ignored":["message.keyword"],"_source":{"@timestamp":"2023-11-09T22:23:23.245645495Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 22:23:23.245+00:00 Received response from handler topics=\"codex\" + tid=1 status=200 meth=GET peer=10.244.0.212:38602 uri=/api/codex/v1/debug/peer/16Uiu2HAkwmYFvu5wWJqvd85yPk3aSX4Q6rehifJLnxHbjDpYW64F + content_type=text/plain content_size=142 count=74","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568603245]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2eUutosBOvK4VyD14SuG","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:24.864274139Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:24.863+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=75","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568604864]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WeUutosBOvK4VyD1_Yao","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:27.865537804Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:27.865+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=76","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568607865]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WuUutosBOvK4VyD1_Yao","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:30.867087591Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:30.866+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=77","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568610867]},{"_index":"continuous-tests-pods-2023.11.09","_id":"W-UutosBOvK4VyD1_Yao","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:33.868823411Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:33.868+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=78","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568613868]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JOUvtosBOvK4VyD1HcPO","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:36.870781414Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:36.870+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=79","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568616870]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JeUvtosBOvK4VyD1HcPO","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:39.872202906Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:39.871+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=80","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568619872]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BOUvtosBOvK4VyD1Nfjp","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:42.872795874Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:42.872+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=81","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568622872]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BeUvtosBOvK4VyD1Nfjp","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:45.874057505Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:45.873+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=82","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568625874]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UuYvtosBOvK4VyD1TRL_","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:48.874969981Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:48.874+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=83","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568628874]},{"_index":"continuous-tests-pods-2023.11.09","_id":"U-YvtosBOvK4VyD1TRL_","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:51.880368066Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:51.880+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=84","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568631880]},{"_index":"continuous-tests-pods-2023.11.09","_id":"v-YvtosBOvK4VyD1ZjAZ","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:54.881743484Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:54.881+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=85","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568634881]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wOYvtosBOvK4VyD1ZjAZ","_score":null,"_source":{"@timestamp":"2023-11-09T22:23:57.883281262Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:23:57.882+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=86","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568637883]},{"_index":"continuous-tests-pods-2023.11.09","_id":"quYvtosBOvK4VyD1fl0z","_score":null,"_source":{"@timestamp":"2023-11-09T22:24:00.885303406Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:24:00.884+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=87","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568640885]},{"_index":"continuous-tests-pods-2023.11.09","_id":"q-YvtosBOvK4VyD1fl0z","_score":null,"_source":{"@timestamp":"2023-11-09T22:24:03.886648573Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:24:03.886+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=88","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568643886]},{"_index":"continuous-tests-pods-2023.11.09","_id":"F-YvtosBOvK4VyD1lpdL","_score":null,"_source":{"@timestamp":"2023-11-09T22:24:06.887823106Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:24:06.887+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=89","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568646887]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GOYvtosBOvK4VyD1lpdL","_score":null,"_source":{"@timestamp":"2023-11-09T22:24:09.889149408Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:24:09.888+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=90","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568649889]},{"_index":"continuous-tests-pods-2023.11.09","_id":"M-YvtosBOvK4VyD1qrln","_score":null,"_source":{"@timestamp":"2023-11-09T22:24:12.890414548Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:24:12.890+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=91","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568652890]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NOYvtosBOvK4VyD1qrln","_score":null,"_source":{"@timestamp":"2023-11-09T22:24:15.892441775Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:24:15.892+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=92","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568655892]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JOYvtosBOvK4VyD1wtZ_","_score":null,"_source":{"@timestamp":"2023-11-09T22:24:18.893734761Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:24:18.893+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=93","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568658893]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JeYvtosBOvK4VyD1wtZ_","_score":null,"_source":{"@timestamp":"2023-11-09T22:24:21.895505548Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:24:21.895+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=94","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568661895]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cOcvtosBOvK4VyD12heZ","_score":null,"_source":{"@timestamp":"2023-11-09T22:24:24.896362337Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:24:24.896+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=95","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568664896]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cecvtosBOvK4VyD12heZ","_score":null,"_source":{"@timestamp":"2023-11-09T22:24:27.897521972Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:24:27.897+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=96","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568667897]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fucvtosBOvK4VyD18muw","_score":null,"_source":{"@timestamp":"2023-11-09T22:24:30.898713618Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:24:30.898+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=97","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568670898]},{"_index":"continuous-tests-pods-2023.11.09","_id":"f-cvtosBOvK4VyD18muw","_score":null,"_source":{"@timestamp":"2023-11-09T22:24:33.900184964Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:24:33.899+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=98","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568673900]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZOcwtosBOvK4VyD1Cr3F","_score":null,"_source":{"@timestamp":"2023-11-09T22:24:36.901438879Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:24:36.901+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=99","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568676901]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZecwtosBOvK4VyD1Cr3F","_score":null,"_source":{"@timestamp":"2023-11-09T22:24:39.903645963Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:24:39.903+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=100","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568679903]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_OgwtosBOvK4VyD1IhDd","_score":null,"_source":{"@timestamp":"2023-11-09T22:24:42.904802335Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:24:42.904+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=101","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568682904]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_egwtosBOvK4VyD1IhDd","_score":null,"_source":{"@timestamp":"2023-11-09T22:24:45.906594109Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:24:45.906+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=102","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568685906]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gugwtosBOvK4VyD1NlXz","_score":null,"_source":{"@timestamp":"2023-11-09T22:24:48.908288868Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:24:48.907+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=103","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568688908]},{"_index":"continuous-tests-pods-2023.11.09","_id":"g-gwtosBOvK4VyD1NlXz","_score":null,"_source":{"@timestamp":"2023-11-09T22:24:51.913652751Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:24:51.913+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=104","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568691913]},{"_index":"continuous-tests-pods-2023.11.09","_id":"F-gwtosBOvK4VyD1T3kM","_score":null,"_source":{"@timestamp":"2023-11-09T22:24:54.915248468Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:24:54.914+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=105","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568694915]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GOgwtosBOvK4VyD1T3kM","_score":null,"_source":{"@timestamp":"2023-11-09T22:24:57.916431196Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:24:57.916+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=106","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568697916]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EOgwtosBOvK4VyD1Z5Mh","_score":null,"_source":{"@timestamp":"2023-11-09T22:25:00.918222163Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:25:00.917+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=107","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568700918]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EegwtosBOvK4VyD1Z5Mh","_score":null,"_source":{"@timestamp":"2023-11-09T22:25:03.920310174Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:25:03.919+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=108","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568703920]},{"_index":"continuous-tests-pods-2023.11.09","_id":"T-gwtosBOvK4VyD1f646","_score":null,"_source":{"@timestamp":"2023-11-09T22:25:06.921908790Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:25:06.921+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=109","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568706921]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UOgwtosBOvK4VyD1f646","_score":null,"_source":{"@timestamp":"2023-11-09T22:25:09.923078545Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:25:09.922+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=110","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568709923]},{"_index":"continuous-tests-pods-2023.11.09","_id":"COgwtosBOvK4VyD1l9NT","_score":null,"_source":{"@timestamp":"2023-11-09T22:25:12.924484218Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:25:12.924+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=111","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568712924]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CegwtosBOvK4VyD1l9NT","_score":null,"_source":{"@timestamp":"2023-11-09T22:25:15.925448456Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:25:15.925+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=112","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568715925]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zekwtosBOvK4VyD1rxRu","_score":null,"_source":{"@timestamp":"2023-11-09T22:25:18.926167326Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:25:18.925+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=113","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568718926]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zukwtosBOvK4VyD1rxRu","_score":null,"_source":{"@timestamp":"2023-11-09T22:25:21.927469188Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:25:21.927+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=114","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568721927]},{"_index":"continuous-tests-pods-2023.11.09","_id":"z-kwtosBOvK4VyD1rxRu","_score":null,"_source":{"@timestamp":"2023-11-09T22:25:24.933458270Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:25:24.933+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=115","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568724933]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MOkwtosBOvK4VyD1z4Ca","_score":null,"_source":{"@timestamp":"2023-11-09T22:25:27.935350866Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:25:27.935+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=116","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568727935]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MekwtosBOvK4VyD1z4Ca","_score":null,"_source":{"@timestamp":"2023-11-09T22:25:30.936536298Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:25:30.936+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=117","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568730936]},{"_index":"continuous-tests-pods-2023.11.09","_id":"x-kwtosBOvK4VyD1566u","_score":null,"_source":{"@timestamp":"2023-11-09T22:25:33.938271385Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:25:33.937+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=118","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568733938]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yOkwtosBOvK4VyD1566u","_score":null,"_source":{"@timestamp":"2023-11-09T22:25:36.939883008Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:25:36.939+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=119","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568736939]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6ukwtosBOvK4VyD1_-fF","_score":null,"_source":{"@timestamp":"2023-11-09T22:25:39.941072530Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:25:39.940+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=120","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568739941]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6-kwtosBOvK4VyD1_-fF","_score":null,"_source":{"@timestamp":"2023-11-09T22:25:42.942976370Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:25:42.942+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=121","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568742942]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4OoxtosBOvK4VyD1FwPc","_score":null,"_source":{"@timestamp":"2023-11-09T22:25:45.948936783Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:25:45.948+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=122","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568745948]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4eoxtosBOvK4VyD1FwPc","_score":null,"_source":{"@timestamp":"2023-11-09T22:25:48.950436503Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:25:48.950+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=123","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568748950]},{"_index":"continuous-tests-pods-2023.11.09","_id":"M-oxtosBOvK4VyD1Lx71","_score":null,"_source":{"@timestamp":"2023-11-09T22:25:51.952418507Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:25:51.952+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=124","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568751952]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NOoxtosBOvK4VyD1Lx71","_score":null,"_source":{"@timestamp":"2023-11-09T22:25:54.954380583Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:25:54.954+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=125","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568754954]},{"_index":"continuous-tests-pods-2023.11.09","_id":"b-oxtosBOvK4VyD1SD4p","_score":null,"_source":{"@timestamp":"2023-11-09T22:25:57.956175353Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:25:57.955+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=126","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568757956]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cOoxtosBOvK4VyD1SD4p","_score":null,"_source":{"@timestamp":"2023-11-09T22:26:00.957486947Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:26:00.957+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=127","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568760957]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JuoxtosBOvK4VyD1XGgq","_score":null,"_source":{"@timestamp":"2023-11-09T22:26:03.959643325Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:26:03.959+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=128","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568763959]},{"_index":"continuous-tests-pods-2023.11.09","_id":"J-oxtosBOvK4VyD1XGgq","_score":null,"_source":{"@timestamp":"2023-11-09T22:26:06.960239259Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:26:06.959+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=129","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568766960]},{"_index":"continuous-tests-pods-2023.11.09","_id":"U-oxtosBOvK4VyD1dJ5A","_score":null,"_source":{"@timestamp":"2023-11-09T22:26:09.961568694Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:26:09.961+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=130","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568769961]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VOoxtosBOvK4VyD1dJ5A","_score":null,"_source":{"@timestamp":"2023-11-09T22:26:12.963242736Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:26:12.962+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=131","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568772963]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1-oxtosBOvK4VyD1jMFZ","_score":null,"_source":{"@timestamp":"2023-11-09T22:26:15.964128892Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:26:15.963+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=132","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568775964]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2OoxtosBOvK4VyD1jMFZ","_score":null,"_source":{"@timestamp":"2023-11-09T22:26:18.965797568Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:26:18.965+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=133","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568778965]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NeoxtosBOvK4VyD1pOBz","_score":null,"_source":{"@timestamp":"2023-11-09T22:26:21.966457011Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:26:21.966+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=134","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568781966]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NuoxtosBOvK4VyD1pOBz","_score":null,"_source":{"@timestamp":"2023-11-09T22:26:24.967799882Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:26:24.967+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=135","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568784967]},{"_index":"continuous-tests-pods-2023.11.09","_id":"w-sxtosBOvK4VyD1vDKG","_score":null,"_source":{"@timestamp":"2023-11-09T22:26:27.969856341Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:26:27.969+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=136","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568787969]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xOsxtosBOvK4VyD1vDKG","_score":null,"_source":{"@timestamp":"2023-11-09T22:26:30.972007619Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:26:30.971+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=137","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568790972]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jesxtosBOvK4VyD11IWc","_score":null,"_source":{"@timestamp":"2023-11-09T22:26:33.973921476Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:26:33.973+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=138","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568793973]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jusxtosBOvK4VyD11IWc","_score":null,"_source":{"@timestamp":"2023-11-09T22:26:36.974777268Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:26:36.974+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=139","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568796974]},{"_index":"continuous-tests-pods-2023.11.09","_id":"j-sxtosBOvK4VyD11IWc","_score":null,"_source":{"@timestamp":"2023-11-09T22:26:39.976450792Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:26:39.976+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=140","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568799976]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DOsxtosBOvK4VyD19PW-","_score":null,"_source":{"@timestamp":"2023-11-09T22:26:42.978747328Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:26:42.978+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=141","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568802978]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DesxtosBOvK4VyD19PW-","_score":null,"_source":{"@timestamp":"2023-11-09T22:26:45.980581461Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:26:45.980+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=142","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568805980]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HOwytosBOvK4VyD1DETX","_score":null,"_source":{"@timestamp":"2023-11-09T22:26:48.982532810Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:26:48.982+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=143","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568808982]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HewytosBOvK4VyD1DETX","_score":null,"_source":{"@timestamp":"2023-11-09T22:26:51.983874675Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:26:51.983+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=144","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568811983]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pewytosBOvK4VyD1JHLs","_score":null,"_source":{"@timestamp":"2023-11-09T22:26:54.985537323Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:26:54.985+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=145","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568814985]},{"_index":"continuous-tests-pods-2023.11.09","_id":"puwytosBOvK4VyD1JHLs","_score":null,"_source":{"@timestamp":"2023-11-09T22:26:57.987169035Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:26:57.986+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=146","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568817987]},{"_index":"continuous-tests-pods-2023.11.09","_id":"V-wytosBOvK4VyD1PY4F","_score":null,"_source":{"@timestamp":"2023-11-09T22:27:00.988292291Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:27:00.987+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=147","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568820988]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WOwytosBOvK4VyD1PY4F","_score":null,"_source":{"@timestamp":"2023-11-09T22:27:03.990259430Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:27:03.989+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=148","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568823990]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MewytosBOvK4VyD1Vasd","_score":null,"_source":{"@timestamp":"2023-11-09T22:27:06.992158915Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:27:06.991+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=149","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568826992]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MuwytosBOvK4VyD1Vasd","_score":null,"_source":{"@timestamp":"2023-11-09T22:27:09.993910627Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:27:09.993+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=150","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568829993]},{"_index":"continuous-tests-pods-2023.11.09","_id":"u-wytosBOvK4VyD1bc49","_score":null,"_source":{"@timestamp":"2023-11-09T22:27:12.995955625Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:27:12.995+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=151","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568832995]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vOwytosBOvK4VyD1bc49","_score":null,"_source":{"@timestamp":"2023-11-09T22:27:15.997516745Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:27:15.997+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=152","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568835997]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ae0ytosBOvK4VyD1gQNI","_score":null,"_source":{"@timestamp":"2023-11-09T22:27:18.998739765Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:27:18.998+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=153","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568838998]},{"_index":"continuous-tests-pods-2023.11.09","_id":"au0ytosBOvK4VyD1gQNI","_score":null,"_source":{"@timestamp":"2023-11-09T22:27:22.002267225Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:27:22.001+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=154","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568842002]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_-0ytosBOvK4VyD1mU1h","_score":null,"_source":{"@timestamp":"2023-11-09T22:27:25.002750700Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:27:25.002+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=155","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568845002]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AO0ytosBOvK4VyD1mU5h","_score":null,"_source":{"@timestamp":"2023-11-09T22:27:28.003013892Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:27:28.002+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=156","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568848003]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ke0ytosBOvK4VyD1sZeA","_score":null,"_source":{"@timestamp":"2023-11-09T22:27:31.004278678Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:27:31.003+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=157","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568851004]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ku0ytosBOvK4VyD1sZeA","_score":null,"_source":{"@timestamp":"2023-11-09T22:27:34.004951503Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:27:34.004+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=158","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568854004]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1-0ytosBOvK4VyD1ydeP","_score":null,"_source":{"@timestamp":"2023-11-09T22:27:37.010941904Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:27:37.010+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=159","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568857010]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2O0ytosBOvK4VyD1ydeP","_score":null,"_source":{"@timestamp":"2023-11-09T22:27:40.011933747Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:27:40.011+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=160","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568860011]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1u0ytosBOvK4VyD14fGl","_score":null,"_source":{"@timestamp":"2023-11-09T22:27:43.013130620Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:27:43.012+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=161","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568863013]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1-0ytosBOvK4VyD14fGl","_score":null,"_source":{"@timestamp":"2023-11-09T22:27:46.014189353Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:27:46.013+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=162","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568866014]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xO4ytosBOvK4VyD1-Q69","_score":null,"_source":{"@timestamp":"2023-11-09T22:27:49.014889733Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:27:49.014+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=163","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568869014]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xe4ytosBOvK4VyD1-Q69","_score":null,"_source":{"@timestamp":"2023-11-09T22:27:52.015425748Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:27:52.015+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=164","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568872015]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xu4ytosBOvK4VyD1-Q69","_score":null,"_source":{"@timestamp":"2023-11-09T22:27:55.015987921Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:27:55.015+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=165","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568875015]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Oe4ztosBOvK4VyD1GTPg","_score":null,"_source":{"@timestamp":"2023-11-09T22:27:58.017305159Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:27:58.016+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=166","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568878017]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ou4ztosBOvK4VyD1GTPg","_score":null,"_source":{"@timestamp":"2023-11-09T22:28:01.017722586Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:28:01.017+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=167","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568881017]},{"_index":"continuous-tests-pods-2023.11.09","_id":"B-4ztosBOvK4VyD1MWr7","_score":null,"_source":{"@timestamp":"2023-11-09T22:28:03.852627873Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:28:03.851+00:00 Cleaning up expired records tid=1 + count=168","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568883852]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CO4ztosBOvK4VyD1MWr7","_score":null,"_source":{"@timestamp":"2023-11-09T22:28:03.852676295Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:28:03.852+00:00 Cleaned up expired records tid=1 + size=0 count=169","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568883852]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ce4ztosBOvK4VyD1MWr7","_score":null,"_source":{"@timestamp":"2023-11-09T22:28:03.852681498Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:28:03.852+00:00 Cleaning up query iterator tid=1 + count=170","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568883852]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Cu4ztosBOvK4VyD1MWr7","_score":null,"_source":{"@timestamp":"2023-11-09T22:28:03.852685763Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:28:03.852+00:00 Cleaning up orphaned records tid=1 + count=171","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568883852]},{"_index":"continuous-tests-pods-2023.11.09","_id":"C-4ztosBOvK4VyD1MWr7","_score":null,"_source":{"@timestamp":"2023-11-09T22:28:03.852690727Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:28:03.852+00:00 Cleaning up query iterator tid=1 + count=172","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568883852]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DO4ztosBOvK4VyD1MWr7","_score":null,"_source":{"@timestamp":"2023-11-09T22:28:04.023771039Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:28:04.023+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=173","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568884023]},{"_index":"continuous-tests-pods-2023.11.09","_id":"De4ztosBOvK4VyD1MWr7","_score":null,"_source":{"@timestamp":"2023-11-09T22:28:07.024711747Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:28:07.024+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=174","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568887024]},{"_index":"continuous-tests-pods-2023.11.09","_id":"G-4ztosBOvK4VyD1SqUS","_score":null,"_source":{"@timestamp":"2023-11-09T22:28:10.025608504Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:28:10.025+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=175","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568890025]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HO4ztosBOvK4VyD1SqUS","_score":null,"_source":{"@timestamp":"2023-11-09T22:28:13.026764556Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:28:13.026+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=176","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568893026]},{"_index":"continuous-tests-pods-2023.11.09","_id":"j-4ztosBOvK4VyD1Yr8r","_score":null,"_source":{"@timestamp":"2023-11-09T22:28:16.027459419Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:28:16.027+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=177","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568896027]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kO4ztosBOvK4VyD1Yr8r","_score":null,"_source":{"@timestamp":"2023-11-09T22:28:19.028430542Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:28:19.028+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=178","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568899028]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Fu4ztosBOvK4VyD1eu9D","_score":null,"_source":{"@timestamp":"2023-11-09T22:28:22.029717554Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:28:22.029+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=179","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568902029]},{"_index":"continuous-tests-pods-2023.11.09","_id":"F-4ztosBOvK4VyD1eu9D","_score":null,"_source":{"@timestamp":"2023-11-09T22:28:25.034537800Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:28:25.034+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=180","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568905034]},{"_index":"continuous-tests-pods-2023.11.09","_id":"He8ztosBOvK4VyD1kkFZ","_score":null,"_source":{"@timestamp":"2023-11-09T22:28:28.034752422Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:28:28.034+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=181","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568908034]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Hu8ztosBOvK4VyD1kkFZ","_score":null,"_source":{"@timestamp":"2023-11-09T22:28:31.035356283Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:28:31.035+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=182","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568911035]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZO8ztosBOvK4VyD1poVw","_score":null,"_source":{"@timestamp":"2023-11-09T22:28:34.036335634Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:28:34.036+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=183","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568914036]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ze8ztosBOvK4VyD1poVw","_score":null,"_source":{"@timestamp":"2023-11-09T22:28:37.037332859Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:28:37.037+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=184","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568917037]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Z-8ztosBOvK4VyD1vtaH","_score":null,"_source":{"@timestamp":"2023-11-09T22:28:40.038321900Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:28:40.037+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=185","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568920038]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aO8ztosBOvK4VyD1vtaH","_score":null,"_source":{"@timestamp":"2023-11-09T22:28:43.038713642Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:28:43.038+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=186","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568923038]},{"_index":"continuous-tests-pods-2023.11.09","_id":"c_AztosBOvK4VyD11iug","_score":null,"_source":{"@timestamp":"2023-11-09T22:28:46.039252597Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:28:46.038+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=187","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568926039]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dPAztosBOvK4VyD11iug","_score":null,"_source":{"@timestamp":"2023-11-09T22:28:49.039753372Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:28:49.039+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=188","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568929039]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4_AztosBOvK4VyD17mK3","_score":null,"_source":{"@timestamp":"2023-11-09T22:28:52.040895242Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:28:52.040+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=189","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568932040]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5PAztosBOvK4VyD17mK3","_score":null,"_source":{"@timestamp":"2023-11-09T22:28:55.041495856Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:28:55.041+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=190","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568935041]},{"_index":"continuous-tests-pods-2023.11.09","_id":"APA0tosBOvK4VyD1Bn_R","_score":null,"_source":{"@timestamp":"2023-11-09T22:28:58.042283979Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:28:58.041+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=191","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568938042]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AfA0tosBOvK4VyD1Bn_R","_score":null,"_source":{"@timestamp":"2023-11-09T22:29:01.042980899Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:29:01.042+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=192","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568941042]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1vA0tosBOvK4VyD1Hprp","_score":null,"_source":{"@timestamp":"2023-11-09T22:29:04.044064753Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:29:04.043+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=193","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568944044]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1_A0tosBOvK4VyD1Hprp","_score":null,"_source":{"@timestamp":"2023-11-09T22:29:07.049038636Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:29:07.048+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=194","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568947049]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oPA0tosBOvK4VyD1M7EC","_score":null,"_source":{"@timestamp":"2023-11-09T22:29:10.049485789Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:29:10.049+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=195","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568950049]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ofA0tosBOvK4VyD1M7EC","_score":null,"_source":{"@timestamp":"2023-11-09T22:29:13.050680399Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:29:13.050+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=196","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568953050]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1PA0tosBOvK4VyD1S9sb","_score":null,"_source":{"@timestamp":"2023-11-09T22:29:16.051215737Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:29:16.050+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=197","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568956051]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1fA0tosBOvK4VyD1S9sb","_score":null,"_source":{"@timestamp":"2023-11-09T22:29:19.051760236Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:29:19.051+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=198","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568959051]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RfE0tosBOvK4VyD1YyUz","_score":null,"_source":{"@timestamp":"2023-11-09T22:29:22.052197229Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:29:22.051+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=199","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568962052]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RvE0tosBOvK4VyD1YyUz","_score":null,"_source":{"@timestamp":"2023-11-09T22:29:25.053021827Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:29:25.052+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=200","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568965053]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZvE0tosBOvK4VyD1e3FL","_score":null,"_source":{"@timestamp":"2023-11-09T22:29:28.058366451Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:29:28.058+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=201","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568968058]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Z_E0tosBOvK4VyD1e3FL","_score":null,"_source":{"@timestamp":"2023-11-09T22:29:31.059526638Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:29:31.059+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=202","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568971059]},{"_index":"continuous-tests-pods-2023.11.09","_id":"D_E0tosBOvK4VyD1k61n","_score":null,"_source":{"@timestamp":"2023-11-09T22:29:34.060158299Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:29:34.059+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=203","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568974060]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EPE0tosBOvK4VyD1k61n","_score":null,"_source":{"@timestamp":"2023-11-09T22:29:37.060857527Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:29:37.060+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=204","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568977060]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vfE0tosBOvK4VyD1q-F9","_score":null,"_source":{"@timestamp":"2023-11-09T22:29:40.061202960Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:29:40.060+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=205","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568980061]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vvE0tosBOvK4VyD1q-F9","_score":null,"_source":{"@timestamp":"2023-11-09T22:29:43.062003173Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:29:43.061+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=206","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568983062]},{"_index":"continuous-tests-pods-2023.11.09","_id":"v_E0tosBOvK4VyD1q-F9","_score":null,"_source":{"@timestamp":"2023-11-09T22:29:46.062671488Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:29:46.062+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=207","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568986062]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gPI0tosBOvK4VyD1ywWh","_score":null,"_source":{"@timestamp":"2023-11-09T22:29:49.067828167Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:29:49.067+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=208","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568989067]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gfI0tosBOvK4VyD1ywWh","_score":null,"_source":{"@timestamp":"2023-11-09T22:29:52.068503312Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:29:52.068+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=209","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568992068]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4_I0tosBOvK4VyD14yC_","_score":null,"_source":{"@timestamp":"2023-11-09T22:29:55.069374176Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:29:55.069+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=210","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568995069]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5PI0tosBOvK4VyD14yC_","_score":null,"_source":{"@timestamp":"2023-11-09T22:29:58.070532801Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:29:58.070+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=211","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699568998070]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3_I0tosBOvK4VyD1-0LP","_score":null,"_source":{"@timestamp":"2023-11-09T22:30:01.071321471Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:30:01.070+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=212","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569001071]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4PI0tosBOvK4VyD1-0LP","_score":null,"_source":{"@timestamp":"2023-11-09T22:30:04.072598608Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:30:04.072+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=213","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569004072]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qvI1tosBOvK4VyD1E3Tm","_score":null,"_source":{"@timestamp":"2023-11-09T22:30:07.073678595Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:30:07.073+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=214","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569007073]},{"_index":"continuous-tests-pods-2023.11.09","_id":"q_I1tosBOvK4VyD1E3Tm","_score":null,"_source":{"@timestamp":"2023-11-09T22:30:10.079524422Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:30:10.079+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=215","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569010079]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wPI1tosBOvK4VyD1LKgA","_score":null,"_source":{"@timestamp":"2023-11-09T22:30:13.080028061Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:30:13.079+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=216","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569013080]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wfI1tosBOvK4VyD1LKgA","_score":null,"_source":{"@timestamp":"2023-11-09T22:30:16.080550858Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:30:16.080+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=217","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569016080]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rvI1tosBOvK4VyD1RMgZ","_score":null,"_source":{"@timestamp":"2023-11-09T22:30:19.081858183Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:30:19.081+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=218","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569019081]},{"_index":"continuous-tests-pods-2023.11.09","_id":"r_I1tosBOvK4VyD1RMgZ","_score":null,"_source":{"@timestamp":"2023-11-09T22:30:22.083096388Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:30:22.082+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=219","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569022083]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cPI1tosBOvK4VyD1WOEy","_score":null,"_source":{"@timestamp":"2023-11-09T22:30:25.084122189Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:30:25.083+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=220","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569025084]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cfI1tosBOvK4VyD1WOEy","_score":null,"_source":{"@timestamp":"2023-11-09T22:30:28.085232990Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:30:28.084+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=221","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569028085]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cfM1tosBOvK4VyD1cDRJ","_score":null,"_source":{"@timestamp":"2023-11-09T22:30:31.090851742Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:30:31.090+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=222","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569031090]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cvM1tosBOvK4VyD1cDRJ","_score":null,"_source":{"@timestamp":"2023-11-09T22:30:34.091585548Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:30:34.091+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=223","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569034091]},{"_index":"continuous-tests-pods-2023.11.09","_id":"G_M1tosBOvK4VyD1iIhh","_score":null,"_source":{"@timestamp":"2023-11-09T22:30:37.091850940Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:30:37.091+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=224","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569037091]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HPM1tosBOvK4VyD1iIhh","_score":null,"_source":{"@timestamp":"2023-11-09T22:30:40.092360859Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:30:40.091+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=225","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569040092]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MfM1tosBOvK4VyD1oN17","_score":null,"_source":{"@timestamp":"2023-11-09T22:30:43.093169306Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:30:43.092+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=226","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569043093]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MvM1tosBOvK4VyD1oN17","_score":null,"_source":{"@timestamp":"2023-11-09T22:30:46.094231609Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:30:46.093+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=227","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569046094]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5PQ1tosBOvK4VyD1uDOT","_score":null,"_source":{"@timestamp":"2023-11-09T22:30:49.094906325Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:30:49.094+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=228","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569049094]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5fQ1tosBOvK4VyD1uDOT","_score":null,"_source":{"@timestamp":"2023-11-09T22:30:52.095452332Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:30:52.095+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=229","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569052095]},{"_index":"continuous-tests-pods-2023.11.09","_id":"l_Q1tosBOvK4VyD10G2p","_score":null,"_source":{"@timestamp":"2023-11-09T22:30:55.096212589Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:30:55.095+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=230","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569055096]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mPQ1tosBOvK4VyD10G2p","_score":null,"_source":{"@timestamp":"2023-11-09T22:30:58.096918560Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:30:58.096+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=231","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569058096]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mfQ1tosBOvK4VyD10G2p","_score":null,"_source":{"@timestamp":"2023-11-09T22:31:01.097753099Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:31:01.097+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=232","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569061097]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nfQ1tosBOvK4VyD18JDR","_score":null,"_source":{"@timestamp":"2023-11-09T22:31:04.099156472Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:31:04.098+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=233","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569064099]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nvQ1tosBOvK4VyD18JDR","_score":null,"_source":{"@timestamp":"2023-11-09T22:31:07.099623833Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:31:07.099+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=234","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569067099]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nfQ2tosBOvK4VyD1CKzx","_score":null,"_source":{"@timestamp":"2023-11-09T22:31:10.104924248Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:31:10.104+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=235","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569070104]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nvQ2tosBOvK4VyD1CKzx","_score":null,"_source":{"@timestamp":"2023-11-09T22:31:13.106197838Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:31:13.105+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=236","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569073106]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mPQ2tosBOvK4VyD1IdsJ","_score":null,"_source":{"@timestamp":"2023-11-09T22:31:16.107146667Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:31:16.106+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=237","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569076107]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mfQ2tosBOvK4VyD1IdsJ","_score":null,"_source":{"@timestamp":"2023-11-09T22:31:19.108482450Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:31:19.107+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=238","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569079108]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gPU2tosBOvK4VyD1OSQh","_score":null,"_source":{"@timestamp":"2023-11-09T22:31:22.109251073Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:31:22.108+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=239","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569082109]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gfU2tosBOvK4VyD1OSQh","_score":null,"_source":{"@timestamp":"2023-11-09T22:31:25.110135485Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:31:25.109+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=240","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569085110]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HPU2tosBOvK4VyD1UX83","_score":null,"_source":{"@timestamp":"2023-11-09T22:31:28.110690301Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:31:28.110+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=241","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569088110]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HfU2tosBOvK4VyD1UX83","_score":null,"_source":{"@timestamp":"2023-11-09T22:31:31.111728433Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:31:31.111+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=242","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569091111]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YvU2tosBOvK4VyD1abVN","_score":null,"_source":{"@timestamp":"2023-11-09T22:31:34.112891004Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:31:34.112+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=243","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569094112]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Y_U2tosBOvK4VyD1abVN","_score":null,"_source":{"@timestamp":"2023-11-09T22:31:37.113701451Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:31:37.113+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=244","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569097113]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xvU2tosBOvK4VyD1fdlm","_score":null,"_source":{"@timestamp":"2023-11-09T22:31:40.114970470Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:31:40.114+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=245","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569100114]},{"_index":"continuous-tests-pods-2023.11.09","_id":"x_U2tosBOvK4VyD1fdlm","_score":null,"_source":{"@timestamp":"2023-11-09T22:31:43.116119928Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:31:43.115+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=246","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569103116]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QvU2tosBOvK4VyD1lfWB","_score":null,"_source":{"@timestamp":"2023-11-09T22:31:46.116934548Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:31:46.116+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=247","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569106116]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Q_U2tosBOvK4VyD1lfWB","_score":null,"_source":{"@timestamp":"2023-11-09T22:31:49.118212269Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:31:49.117+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=248","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569109118]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GvY2tosBOvK4VyD1rQ-i","_score":null,"_source":{"@timestamp":"2023-11-09T22:31:52.119537428Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:31:52.119+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=249","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569112119]},{"_index":"continuous-tests-pods-2023.11.09","_id":"G_Y2tosBOvK4VyD1rQ-i","_score":null,"_source":{"@timestamp":"2023-11-09T22:31:55.120106751Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:31:55.119+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=250","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569115120]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bPY2tosBOvK4VyD1xSu1","_score":null,"_source":{"@timestamp":"2023-11-09T22:31:58.121317736Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:31:58.120+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=251","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569118121]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bfY2tosBOvK4VyD1xSu1","_score":null,"_source":{"@timestamp":"2023-11-09T22:32:01.122536927Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:32:01.122+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=252","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569121122]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1_Y2tosBOvK4VyD13WTM","_score":null,"_source":{"@timestamp":"2023-11-09T22:32:04.123299204Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:32:04.122+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=253","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569124123]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2PY2tosBOvK4VyD13WTM","_score":null,"_source":{"@timestamp":"2023-11-09T22:32:07.124392215Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:32:07.124+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=254","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569127124]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RPY2tosBOvK4VyD19Zzi","_score":null,"_source":{"@timestamp":"2023-11-09T22:32:10.125070527Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:32:10.124+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=255","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569130125]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RfY2tosBOvK4VyD19Zzi","_score":null,"_source":{"@timestamp":"2023-11-09T22:32:13.125800848Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:32:13.125+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=256","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569133125]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XfY3tosBOvK4VyD1CbP-","_score":null,"_source":{"@timestamp":"2023-11-09T22:32:16.131452803Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:32:16.131+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=257","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569136131]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XvY3tosBOvK4VyD1CbP-","_score":null,"_source":{"@timestamp":"2023-11-09T22:32:19.132217123Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:32:19.131+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=258","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569139132]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NfY3tosBOvK4VyD1It0W","_score":null,"_source":{"@timestamp":"2023-11-09T22:32:22.132766167Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:32:22.132+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=259","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569142132]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NvY3tosBOvK4VyD1It0W","_score":null,"_source":{"@timestamp":"2023-11-09T22:32:25.133231402Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:32:25.132+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=260","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569145133]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fPc3tosBOvK4VyD1OjIw","_score":null,"_source":{"@timestamp":"2023-11-09T22:32:28.134361671Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:32:28.134+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=261","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569148134]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ffc3tosBOvK4VyD1OjIw","_score":null,"_source":{"@timestamp":"2023-11-09T22:32:31.135470927Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:32:31.135+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=262","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569151135]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yPc3tosBOvK4VyD1UoVI","_score":null,"_source":{"@timestamp":"2023-11-09T22:32:34.135822640Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:32:34.135+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=263","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569154135]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yfc3tosBOvK4VyD1UoVI","_score":null,"_source":{"@timestamp":"2023-11-09T22:32:37.137076110Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:32:37.136+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=264","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569157137]},{"_index":"continuous-tests-pods-2023.11.09","_id":"K_c3tosBOvK4VyD1athi","_score":null,"_source":{"@timestamp":"2023-11-09T22:32:40.137512899Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:32:40.137+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=265","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569160137]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LPc3tosBOvK4VyD1athi","_score":null,"_source":{"@timestamp":"2023-11-09T22:32:43.138780916Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:32:43.138+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=266","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569163138]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UPg3tosBOvK4VyD1giF6","_score":null,"_source":{"@timestamp":"2023-11-09T22:32:46.139168228Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:32:46.138+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=267","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569166139]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ufg3tosBOvK4VyD1giF6","_score":null,"_source":{"@timestamp":"2023-11-09T22:32:49.139656639Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:32:49.139+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=268","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569169139]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZPg3tosBOvK4VyD1lleS","_score":null,"_source":{"@timestamp":"2023-11-09T22:32:52.140214365Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:32:52.139+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=269","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569172140]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Zfg3tosBOvK4VyD1lleS","_score":null,"_source":{"@timestamp":"2023-11-09T22:32:55.141409916Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:32:55.141+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=270","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569175141]},{"_index":"continuous-tests-pods-2023.11.09","_id":"R_g3tosBOvK4VyD1rnOs","_score":null,"_source":{"@timestamp":"2023-11-09T22:32:58.142510098Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:32:58.142+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=271","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569178142]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SPg3tosBOvK4VyD1rnOs","_score":null,"_source":{"@timestamp":"2023-11-09T22:33:01.143367723Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:33:01.142+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=272","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569181143]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3vg3tosBOvK4VyD1xo7G","_score":null,"_source":{"@timestamp":"2023-11-09T22:33:03.854048334Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:33:03.853+00:00 Cleaning up expired records tid=1 + count=273","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569183854]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3_g3tosBOvK4VyD1xo7G","_score":null,"_source":{"@timestamp":"2023-11-09T22:33:03.854106194Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:33:03.853+00:00 Cleaned up expired records tid=1 + size=0 count=274","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569183854]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4Pg3tosBOvK4VyD1xo7G","_score":null,"_source":{"@timestamp":"2023-11-09T22:33:03.854112230Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:33:03.853+00:00 Cleaning up query iterator tid=1 + count=275","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569183854]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4fg3tosBOvK4VyD1xo7G","_score":null,"_source":{"@timestamp":"2023-11-09T22:33:03.854117007Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:33:03.853+00:00 Cleaning up orphaned records tid=1 + count=276","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569183854]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4vg3tosBOvK4VyD1xo7G","_score":null,"_source":{"@timestamp":"2023-11-09T22:33:03.854121304Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:33:03.853+00:00 Cleaning up query iterator tid=1 + count=277","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569183854]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4_g3tosBOvK4VyD1xo7G","_score":null,"_source":{"@timestamp":"2023-11-09T22:33:04.144074048Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:33:04.143+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=278","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569184144]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5Pg3tosBOvK4VyD1xo7G","_score":null,"_source":{"@timestamp":"2023-11-09T22:33:07.144753472Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:33:07.144+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=279","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569187144]},{"_index":"continuous-tests-pods-2023.11.09","_id":"M_g3tosBOvK4VyD13qnj","_score":null,"_source":{"@timestamp":"2023-11-09T22:33:10.145284022Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:33:10.144+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=280","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569190145]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NPg3tosBOvK4VyD13qnj","_score":null,"_source":{"@timestamp":"2023-11-09T22:33:13.146564230Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:33:13.146+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=281","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569193146]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Pfg3tosBOvK4VyD19tby","_score":null,"_source":{"@timestamp":"2023-11-09T22:33:16.147730778Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:33:16.147+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=282","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569196147]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Pvg3tosBOvK4VyD19tby","_score":null,"_source":{"@timestamp":"2023-11-09T22:33:19.148925503Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:33:19.148+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=283","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569199148]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Avk4tosBOvK4VyD1DxUI","_score":null,"_source":{"@timestamp":"2023-11-09T22:33:22.150375629Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:33:22.149+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=284","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569202150]},{"_index":"continuous-tests-pods-2023.11.09","_id":"A_k4tosBOvK4VyD1DxUI","_score":null,"_source":{"@timestamp":"2023-11-09T22:33:25.151204200Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:33:25.150+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=285","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569205151]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BPk4tosBOvK4VyD1DxUI","_score":null,"_source":{"@timestamp":"2023-11-09T22:33:28.151858047Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:33:28.151+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=286","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569208151]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Pfk4tosBOvK4VyD1L34t","_score":null,"_source":{"@timestamp":"2023-11-09T22:33:31.152830836Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:33:31.152+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=287","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569211152]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Pvk4tosBOvK4VyD1L34t","_score":null,"_source":{"@timestamp":"2023-11-09T22:33:34.153743709Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:33:34.153+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=288","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569214153]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Pvk4tosBOvK4VyD1R6tl","_score":null,"_source":{"@timestamp":"2023-11-09T22:33:37.154693349Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:33:37.154+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=289","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569217154]},{"_index":"continuous-tests-pods-2023.11.09","_id":"P_k4tosBOvK4VyD1R6tl","_score":null,"_source":{"@timestamp":"2023-11-09T22:33:40.155228416Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:33:40.154+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=290","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569220155]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Jfk4tosBOvK4VyD1X-F8","_score":null,"_source":{"@timestamp":"2023-11-09T22:33:43.156271989Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:33:43.156+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=291","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569223156]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Jvk4tosBOvK4VyD1X-F8","_score":null,"_source":{"@timestamp":"2023-11-09T22:33:46.156896502Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:33:46.156+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=292","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569226156]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6vk4tosBOvK4VyD1d_yX","_score":null,"_source":{"@timestamp":"2023-11-09T22:33:49.158003541Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:33:49.157+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=293","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569229158]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6_k4tosBOvK4VyD1d_yX","_score":null,"_source":{"@timestamp":"2023-11-09T22:33:52.159049357Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:33:52.158+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=294","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569232159]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-fo4tosBOvK4VyD1jxa0","_score":null,"_source":{"@timestamp":"2023-11-09T22:33:55.164899909Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:33:55.164+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=295","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569235164]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-vo4tosBOvK4VyD1jxa0","_score":null,"_source":{"@timestamp":"2023-11-09T22:33:58.165611930Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:33:58.165+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=296","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569238165]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ifo4tosBOvK4VyD1pzfM","_score":null,"_source":{"@timestamp":"2023-11-09T22:34:01.166471620Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:34:01.166+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=297","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569241166]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ivo4tosBOvK4VyD1pzfM","_score":null,"_source":{"@timestamp":"2023-11-09T22:34:04.167067655Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:34:04.166+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=298","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569244167]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Sfo4tosBOvK4VyD1u2Li","_score":null,"_source":{"@timestamp":"2023-11-09T22:34:07.168269056Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:34:07.167+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=299","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569247168]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Svo4tosBOvK4VyD1u2Li","_score":null,"_source":{"@timestamp":"2023-11-09T22:34:10.169443085Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:34:10.169+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=300","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569250169]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Rvo4tosBOvK4VyD1053-","_score":null,"_source":{"@timestamp":"2023-11-09T22:34:13.170750882Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:34:13.170+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=301","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569253170]},{"_index":"continuous-tests-pods-2023.11.09","_id":"R_o4tosBOvK4VyD1053-","_score":null,"_source":{"@timestamp":"2023-11-09T22:34:16.176692685Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:34:16.176+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=302","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569256176]},{"_index":"continuous-tests-pods-2023.11.09","_id":"c_o4tosBOvK4VyD17LwS","_score":null,"_source":{"@timestamp":"2023-11-09T22:34:19.177245618Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:34:19.176+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=303","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569259177]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dPo4tosBOvK4VyD17LwS","_score":null,"_source":{"@timestamp":"2023-11-09T22:34:22.178223178Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:34:22.177+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=304","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569262178]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Dfo5tosBOvK4VyD1BOkt","_score":null,"_source":{"@timestamp":"2023-11-09T22:34:25.178920601Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:34:25.178+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=305","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569265178]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Dvo5tosBOvK4VyD1BOkt","_score":null,"_source":{"@timestamp":"2023-11-09T22:34:28.180157799Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:34:28.179+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=306","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569268180]},{"_index":"continuous-tests-pods-2023.11.09","_id":"K_s5tosBOvK4VyD1HDhD","_score":null,"_source":{"@timestamp":"2023-11-09T22:34:31.181381827Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:34:31.181+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=307","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569271181]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LPs5tosBOvK4VyD1HDhD","_score":null,"_source":{"@timestamp":"2023-11-09T22:34:34.182573692Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:34:34.182+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=308","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569274182]},{"_index":"continuous-tests-pods-2023.11.09","_id":"X_s5tosBOvK4VyD1NIxZ","_score":null,"_source":{"@timestamp":"2023-11-09T22:34:37.187748272Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:34:37.187+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=309","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569277187]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YPs5tosBOvK4VyD1NIxZ","_score":null,"_source":{"@timestamp":"2023-11-09T22:34:40.188596308Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:34:40.188+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=310","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569280188]},{"_index":"continuous-tests-pods-2023.11.09","_id":"o_s5tosBOvK4VyD1SNN0","_score":null,"_source":{"@timestamp":"2023-11-09T22:34:43.189007848Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:34:43.188+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=311","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569283189]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pPs5tosBOvK4VyD1SNN0","_score":null,"_source":{"@timestamp":"2023-11-09T22:34:46.189941966Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:34:46.189+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=312","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569286189]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Wvw5tosBOvK4VyD1YCeJ","_score":null,"_source":{"@timestamp":"2023-11-09T22:34:49.190811173Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:34:49.190+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=313","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569289190]},{"_index":"continuous-tests-pods-2023.11.09","_id":"W_w5tosBOvK4VyD1YCeJ","_score":null,"_source":{"@timestamp":"2023-11-09T22:34:52.191390205Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:34:52.191+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=314","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569292191]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Vvw5tosBOvK4VyD1eGCi","_score":null,"_source":{"@timestamp":"2023-11-09T22:34:55.192108475Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:34:55.191+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=315","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569295192]},{"_index":"continuous-tests-pods-2023.11.09","_id":"V_w5tosBOvK4VyD1eGCi","_score":null,"_source":{"@timestamp":"2023-11-09T22:34:58.197976105Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:34:58.197+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=316","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569298197]},{"_index":"continuous-tests-pods-2023.11.09","_id":"U_w5tosBOvK4VyD1kHm7","_score":null,"_source":{"@timestamp":"2023-11-09T22:35:01.199127224Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:35:01.198+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=317","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569301199]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VPw5tosBOvK4VyD1kHm7","_score":null,"_source":{"@timestamp":"2023-11-09T22:35:04.199606903Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:35:04.199+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=318","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569304199]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mfw5tosBOvK4VyD1qJTT","_score":null,"_source":{"@timestamp":"2023-11-09T22:35:07.200863648Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:35:07.200+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=319","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569307200]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mvw5tosBOvK4VyD1qJTT","_score":null,"_source":{"@timestamp":"2023-11-09T22:35:10.201521494Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:35:10.201+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=320","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569310201]},{"_index":"continuous-tests-pods-2023.11.09","_id":"X_w5tosBOvK4VyD1wLbn","_score":null,"_source":{"@timestamp":"2023-11-09T22:35:13.202019296Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:35:13.201+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=321","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569313202]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YPw5tosBOvK4VyD1wLbn","_score":null,"_source":{"@timestamp":"2023-11-09T22:35:16.203449553Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:35:16.203+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=322","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569316203]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Yfw5tosBOvK4VyD1wLbn","_score":null,"_source":{"@timestamp":"2023-11-09T22:35:19.208947644Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:35:19.208+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=323","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569319208]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nP05tosBOvK4VyD14Q4K","_score":null,"_source":{"@timestamp":"2023-11-09T22:35:22.210208673Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:35:22.209+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=324","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569322210]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nf05tosBOvK4VyD14Q4K","_score":null,"_source":{"@timestamp":"2023-11-09T22:35:25.210970830Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:35:25.210+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=325","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569325210]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WP05tosBOvK4VyD1-WMq","_score":null,"_source":{"@timestamp":"2023-11-09T22:35:28.212220891Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:35:28.211+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=326","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569328212]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Wf05tosBOvK4VyD1-WMq","_score":null,"_source":{"@timestamp":"2023-11-09T22:35:31.213128027Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:35:31.212+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=327","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569331213]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bP06tosBOvK4VyD1EZND","_score":null,"_source":{"@timestamp":"2023-11-09T22:35:34.213738116Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:35:34.213+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=328","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569334213]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bf06tosBOvK4VyD1EZND","_score":null,"_source":{"@timestamp":"2023-11-09T22:35:37.214328410Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:35:37.213+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=329","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569337214]},{"_index":"continuous-tests-pods-2023.11.09","_id":"a_06tosBOvK4VyD1Kc9b","_score":null,"_source":{"@timestamp":"2023-11-09T22:35:40.215227447Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:35:40.214+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=330","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569340215]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bP06tosBOvK4VyD1Kc9b","_score":null,"_source":{"@timestamp":"2023-11-09T22:35:43.215911536Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:35:43.215+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=331","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569343215]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3_06tosBOvK4VyD1Qetx","_score":null,"_source":{"@timestamp":"2023-11-09T22:35:46.216782207Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:35:46.216+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=332","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569346216]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4P06tosBOvK4VyD1Qetx","_score":null,"_source":{"@timestamp":"2023-11-09T22:35:49.217802576Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:35:49.217+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=333","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569349217]},{"_index":"continuous-tests-pods-2023.11.09","_id":"M_46tosBOvK4VyD1WQeH","_score":null,"_source":{"@timestamp":"2023-11-09T22:35:52.218704030Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:35:52.218+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=334","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569352218]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NP46tosBOvK4VyD1WQeH","_score":null,"_source":{"@timestamp":"2023-11-09T22:35:55.220039099Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:35:55.219+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=335","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569355220]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ev46tosBOvK4VyD1bR6i","_score":null,"_source":{"@timestamp":"2023-11-09T22:35:58.221059668Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:35:58.220+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=336","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569358221]},{"_index":"continuous-tests-pods-2023.11.09","_id":"E_46tosBOvK4VyD1bR6i","_score":null,"_source":{"@timestamp":"2023-11-09T22:36:01.226554529Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:36:01.226+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=337","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569361226]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-v46tosBOvK4VyD1hU27","_score":null,"_source":{"@timestamp":"2023-11-09T22:36:04.227827816Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:36:04.227+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=338","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569364227]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-_46tosBOvK4VyD1hU27","_score":null,"_source":{"@timestamp":"2023-11-09T22:36:07.228276128Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:36:07.227+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=339","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569367228]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3v46tosBOvK4VyD1nYXT","_score":null,"_source":{"@timestamp":"2023-11-09T22:36:10.229305495Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:36:10.228+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=340","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569370229]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3_46tosBOvK4VyD1nYXT","_score":null,"_source":{"@timestamp":"2023-11-09T22:36:13.230523868Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:36:13.230+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=341","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569373230]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0v46tosBOvK4VyD1tarn","_score":null,"_source":{"@timestamp":"2023-11-09T22:36:16.231524258Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:36:16.231+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=342","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569376231]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0_46tosBOvK4VyD1tarn","_score":null,"_source":{"@timestamp":"2023-11-09T22:36:19.232937745Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:36:19.232+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=343","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569379232]},{"_index":"continuous-tests-pods-2023.11.09","_id":"r_46tosBOvK4VyD1zsoC","_score":null,"_source":{"@timestamp":"2023-11-09T22:36:22.233650423Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:36:22.233+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=344","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569382233]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sP46tosBOvK4VyD1zsoC","_score":null,"_source":{"@timestamp":"2023-11-09T22:36:25.234178590Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:36:25.233+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=345","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569385234]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qv86tosBOvK4VyD15iAY","_score":null,"_source":{"@timestamp":"2023-11-09T22:36:28.235438825Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:36:28.235+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=346","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569388235]},{"_index":"continuous-tests-pods-2023.11.09","_id":"q_86tosBOvK4VyD15iAY","_score":null,"_source":{"@timestamp":"2023-11-09T22:36:31.235987327Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:36:31.235+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=347","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569391235]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Hv86tosBOvK4VyD1-m0u","_score":null,"_source":{"@timestamp":"2023-11-09T22:36:34.237237328Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:36:34.236+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=348","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569394237]},{"_index":"continuous-tests-pods-2023.11.09","_id":"H_86tosBOvK4VyD1-m0u","_score":null,"_source":{"@timestamp":"2023-11-09T22:36:37.237818195Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:36:37.237+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=349","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569397237]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Wv87tosBOvK4VyD1EsVC","_score":null,"_source":{"@timestamp":"2023-11-09T22:36:40.238369650Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:36:40.238+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=350","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569400238]},{"_index":"continuous-tests-pods-2023.11.09","_id":"W_87tosBOvK4VyD1EsVC","_score":null,"_source":{"@timestamp":"2023-11-09T22:36:43.238860144Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:36:43.238+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=351","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569403238]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SgA7tosBOvK4VyD1Khlb","_score":null,"_source":{"@timestamp":"2023-11-09T22:36:46.239321589Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:36:46.239+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=352","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569406239]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SwA7tosBOvK4VyD1Khlb","_score":null,"_source":{"@timestamp":"2023-11-09T22:36:49.240363156Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:36:49.240+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=353","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569409240]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sAA7tosBOvK4VyD1Qkxy","_score":null,"_source":{"@timestamp":"2023-11-09T22:36:52.240971111Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:36:52.240+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=354","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569412240]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sQA7tosBOvK4VyD1Qkxz","_score":null,"_source":{"@timestamp":"2023-11-09T22:36:55.242193265Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:36:55.241+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=355","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569415242]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EgA7tosBOvK4VyD1WmiH","_score":null,"_source":{"@timestamp":"2023-11-09T22:36:58.243221862Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:36:58.242+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=356","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569418243]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EwA7tosBOvK4VyD1WmiH","_score":null,"_source":{"@timestamp":"2023-11-09T22:37:01.244548965Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:37:01.244+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=357","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569421244]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AAA7tosBOvK4VyD1coKd","_score":null,"_source":{"@timestamp":"2023-11-09T22:37:04.245735729Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:37:04.245+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=358","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569424245]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AQA7tosBOvK4VyD1coKd","_score":null,"_source":{"@timestamp":"2023-11-09T22:37:07.246349763Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:37:07.246+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=359","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569427246]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AgA7tosBOvK4VyD1coKd","_score":null,"_source":{"@timestamp":"2023-11-09T22:37:10.247280784Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:37:10.246+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=360","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569430247]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UQA7tosBOvK4VyD1kqrA","_score":null,"_source":{"@timestamp":"2023-11-09T22:37:13.248727442Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:37:13.248+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=361","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569433248]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UgA7tosBOvK4VyD1kqrA","_score":null,"_source":{"@timestamp":"2023-11-09T22:37:16.250005468Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:37:16.249+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=362","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569436250]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aAA7tosBOvK4VyD1quba","_score":null,"_source":{"@timestamp":"2023-11-09T22:37:19.251058308Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:37:19.250+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=363","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569439251]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aQA7tosBOvK4VyD1quba","_score":null,"_source":{"@timestamp":"2023-11-09T22:37:22.251972095Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:37:22.251+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=364","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569442251]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WwE7tosBOvK4VyD1wi3z","_score":null,"_source":{"@timestamp":"2023-11-09T22:37:25.257048281Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:37:25.256+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=365","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569445257]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XAE7tosBOvK4VyD1wi3z","_score":null,"_source":{"@timestamp":"2023-11-09T22:37:28.258310219Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:37:28.257+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=366","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569448258]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iAE7tosBOvK4VyD123wI","_score":null,"_source":{"@timestamp":"2023-11-09T22:37:31.258917517Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:37:31.258+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=367","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569451258]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iQE7tosBOvK4VyD123wI","_score":null,"_source":{"@timestamp":"2023-11-09T22:37:34.259562711Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:37:34.259+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=368","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569454259]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-wE7tosBOvK4VyD187Ug","_score":null,"_source":{"@timestamp":"2023-11-09T22:37:37.260679456Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:37:37.260+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=369","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569457260]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_AE7tosBOvK4VyD187Ug","_score":null,"_source":{"@timestamp":"2023-11-09T22:37:40.261196369Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:37:40.260+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=370","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569460261]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0gE8tosBOvK4VyD1C9g2","_score":null,"_source":{"@timestamp":"2023-11-09T22:37:43.262760328Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:37:43.262+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=371","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569463262]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0wE8tosBOvK4VyD1C9g2","_score":null,"_source":{"@timestamp":"2023-11-09T22:37:46.267644877Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:37:46.267+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=372","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569466267]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IQE8tosBOvK4VyD1H_FO","_score":null,"_source":{"@timestamp":"2023-11-09T22:37:49.268497798Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:37:49.268+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=373","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569469268]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IgE8tosBOvK4VyD1H_FO","_score":null,"_source":{"@timestamp":"2023-11-09T22:37:52.269266940Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:37:52.268+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=374","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569472269]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XwI8tosBOvK4VyD1Nwto","_score":null,"_source":{"@timestamp":"2023-11-09T22:37:55.270806757Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:37:55.270+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=375","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569475270]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YAI8tosBOvK4VyD1Nwto","_score":null,"_source":{"@timestamp":"2023-11-09T22:37:58.272030995Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:37:58.271+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=376","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569478272]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MwI8tosBOvK4VyD1TzJ_","_score":null,"_source":{"@timestamp":"2023-11-09T22:38:01.273250500Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:38:01.272+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=377","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569481273]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NAI8tosBOvK4VyD1TzJ_","_score":null,"_source":{"@timestamp":"2023-11-09T22:38:03.855243090Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:38:03.854+00:00 Cleaning up expired records tid=1 + count=378","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569483855]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NQI8tosBOvK4VyD1TzJ_","_score":null,"_source":{"@timestamp":"2023-11-09T22:38:03.855327419Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:38:03.855+00:00 Cleaned up expired records tid=1 + size=0 count=379","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569483855]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NgI8tosBOvK4VyD1TzJ_","_score":null,"_source":{"@timestamp":"2023-11-09T22:38:03.855359057Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:38:03.855+00:00 Cleaning up query iterator tid=1 + count=380","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569483855]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NwI8tosBOvK4VyD1TzJ_","_score":null,"_source":{"@timestamp":"2023-11-09T22:38:03.855382040Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:38:03.855+00:00 Cleaning up orphaned records tid=1 + count=381","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569483855]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OAI8tosBOvK4VyD1TzJ_","_score":null,"_source":{"@timestamp":"2023-11-09T22:38:03.855547534Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:38:03.855+00:00 Cleaning up query iterator tid=1 + count=382","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569483855]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OQI8tosBOvK4VyD1TzJ_","_score":null,"_source":{"@timestamp":"2023-11-09T22:38:04.274588025Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:38:04.274+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=383","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569484274]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CgI8tosBOvK4VyD1Z22Y","_score":null,"_source":{"@timestamp":"2023-11-09T22:38:07.279957895Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:38:07.279+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=384","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569487279]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CwI8tosBOvK4VyD1Z22Y","_score":null,"_source":{"@timestamp":"2023-11-09T22:38:10.280984080Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:38:10.280+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=385","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569490280]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FQI8tosBOvK4VyD1f5qt","_score":null,"_source":{"@timestamp":"2023-11-09T22:38:13.282175353Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:38:13.281+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=386","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569493282]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FgI8tosBOvK4VyD1f5qt","_score":null,"_source":{"@timestamp":"2023-11-09T22:38:16.283160528Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:38:16.282+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=387","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569496283]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qQI8tosBOvK4VyD1l7XG","_score":null,"_source":{"@timestamp":"2023-11-09T22:38:19.284662309Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:38:19.284+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=388","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569499284]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qgI8tosBOvK4VyD1l7XG","_score":null,"_source":{"@timestamp":"2023-11-09T22:38:22.285582486Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:38:22.285+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=389","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569502285]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qwI8tosBOvK4VyD1l7XG","_score":null,"_source":{"@timestamp":"2023-11-09T22:38:25.287269397Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:38:25.286+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=390","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569505287]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aQM8tosBOvK4VyD1tx3o","_score":null,"_source":{"@timestamp":"2023-11-09T22:38:28.293020955Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:38:28.292+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=391","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569508293]},{"_index":"continuous-tests-pods-2023.11.09","_id":"agM8tosBOvK4VyD1tx3o","_score":null,"_source":{"@timestamp":"2023-11-09T22:38:31.294244030Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:38:31.293+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=392","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569511294]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nAM8tosBOvK4VyD1z2z_","_score":null,"_source":{"@timestamp":"2023-11-09T22:38:34.295070532Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:38:34.294+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=393","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569514295]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nQM8tosBOvK4VyD1z2z_","_score":null,"_source":{"@timestamp":"2023-11-09T22:38:37.295990383Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:38:37.295+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=394","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569517295]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hAM8tosBOvK4VyD16MAV","_score":null,"_source":{"@timestamp":"2023-11-09T22:38:40.297124869Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:38:40.296+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=395","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569520297]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hQM8tosBOvK4VyD16MAV","_score":null,"_source":{"@timestamp":"2023-11-09T22:38:43.297942823Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:38:43.297+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=396","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569523297]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xAQ9tosBOvK4VyD1AAMt","_score":null,"_source":{"@timestamp":"2023-11-09T22:38:46.299055749Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:38:46.298+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=397","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569526299]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xQQ9tosBOvK4VyD1AAMt","_score":null,"_source":{"@timestamp":"2023-11-09T22:38:49.300259369Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:38:49.299+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=398","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569529300]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_AQ9tosBOvK4VyD1GEpI","_score":null,"_source":{"@timestamp":"2023-11-09T22:38:52.301095473Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:38:52.300+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=399","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569532301]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_QQ9tosBOvK4VyD1GEpI","_score":null,"_source":{"@timestamp":"2023-11-09T22:38:55.302446824Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:38:55.302+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=400","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569535302]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CAQ9tosBOvK4VyD1MGVd","_score":null,"_source":{"@timestamp":"2023-11-09T22:38:58.303737465Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:38:58.303+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=401","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569538303]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CQQ9tosBOvK4VyD1MGVd","_score":null,"_source":{"@timestamp":"2023-11-09T22:39:01.305021872Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:39:01.304+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=402","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569541305]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8AQ9tosBOvK4VyD1RHt-","_score":null,"_source":{"@timestamp":"2023-11-09T22:39:04.305850513Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:39:04.305+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=403","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569544305]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8QQ9tosBOvK4VyD1RHt-","_score":null,"_source":{"@timestamp":"2023-11-09T22:39:07.306426081Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:39:07.306+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=404","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569547306]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uAQ9tosBOvK4VyD1XJeV","_score":null,"_source":{"@timestamp":"2023-11-09T22:39:10.307922131Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:39:10.307+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=405","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569550307]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uQQ9tosBOvK4VyD1XJeV","_score":null,"_source":{"@timestamp":"2023-11-09T22:39:13.309100610Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:39:13.308+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=406","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569553309]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IAQ9tosBOvK4VyD1dLyx","_score":null,"_source":{"@timestamp":"2023-11-09T22:39:16.310345849Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:39:16.310+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=407","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569556310]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IQQ9tosBOvK4VyD1dLyx","_score":null,"_source":{"@timestamp":"2023-11-09T22:39:19.311247660Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:39:19.311+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=408","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569559311]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TAQ9tosBOvK4VyD1jP_I","_score":null,"_source":{"@timestamp":"2023-11-09T22:39:22.317319660Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:39:22.317+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=409","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569562317]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TQQ9tosBOvK4VyD1jP_I","_score":null,"_source":{"@timestamp":"2023-11-09T22:39:25.317970510Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:39:25.317+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=410","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569565317]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2wU9tosBOvK4VyD1pEXg","_score":null,"_source":{"@timestamp":"2023-11-09T22:39:28.319055292Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:39:28.318+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=411","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569568319]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3AU9tosBOvK4VyD1pEXg","_score":null,"_source":{"@timestamp":"2023-11-09T22:39:31.319909434Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:39:31.319+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=412","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569571319]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MwU9tosBOvK4VyD1vIz7","_score":null,"_source":{"@timestamp":"2023-11-09T22:39:34.321191255Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:39:34.320+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=413","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569574321]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NAU9tosBOvK4VyD1vIz7","_score":null,"_source":{"@timestamp":"2023-11-09T22:39:37.321842405Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:39:37.321+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=414","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569577321]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DgU9tosBOvK4VyD10cIR","_score":null,"_source":{"@timestamp":"2023-11-09T22:39:40.323213267Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:39:40.322+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=415","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569580323]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DwU9tosBOvK4VyD10cIR","_score":null,"_source":{"@timestamp":"2023-11-09T22:39:43.328121213Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:39:43.327+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=416","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569583328]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zAU9tosBOvK4VyD16d4q","_score":null,"_source":{"@timestamp":"2023-11-09T22:39:46.328737587Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:39:46.328+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=417","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569586328]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zQU9tosBOvK4VyD16d4q","_score":null,"_source":{"@timestamp":"2023-11-09T22:39:49.329315102Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:39:49.329+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=418","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569589329]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ewU-tosBOvK4VyD1AfpC","_score":null,"_source":{"@timestamp":"2023-11-09T22:39:52.330733098Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:39:52.330+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=419","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569592330]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fAU-tosBOvK4VyD1AfpC","_score":null,"_source":{"@timestamp":"2023-11-09T22:39:55.332195363Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:39:55.331+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=420","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569595332]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZgY-tosBOvK4VyD1GRRb","_score":null,"_source":{"@timestamp":"2023-11-09T22:39:58.332907811Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:39:58.332+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=421","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569598332]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZwY-tosBOvK4VyD1GRRb","_score":null,"_source":{"@timestamp":"2023-11-09T22:40:01.334149823Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:40:01.333+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=422","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569601334]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2QY-tosBOvK4VyD1MT91","_score":null,"_source":{"@timestamp":"2023-11-09T22:40:04.335358314Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:40:04.334+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=423","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569604335]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2gY-tosBOvK4VyD1MT91","_score":null,"_source":{"@timestamp":"2023-11-09T22:40:07.336134910Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:40:07.335+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=424","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569607336]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SAY-tosBOvK4VyD1SXmN","_score":null,"_source":{"@timestamp":"2023-11-09T22:40:10.337056008Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:40:10.336+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=425","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569610337]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SQY-tosBOvK4VyD1SXmN","_score":null,"_source":{"@timestamp":"2023-11-09T22:40:13.338484726Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:40:13.338+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=426","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569613338]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SgY-tosBOvK4VyD1SXmN","_score":null,"_source":{"@timestamp":"2023-11-09T22:40:16.339973531Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:40:16.339+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=427","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569616339]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7QY-tosBOvK4VyD1aaqz","_score":null,"_source":{"@timestamp":"2023-11-09T22:40:19.341033356Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:40:19.340+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=428","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569619341]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7gY-tosBOvK4VyD1aaqz","_score":null,"_source":{"@timestamp":"2023-11-09T22:40:22.341848513Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:40:22.341+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=429","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569622341]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4QY-tosBOvK4VyD1gdjF","_score":null,"_source":{"@timestamp":"2023-11-09T22:40:25.343262589Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:40:25.342+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=430","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569625343]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4gY-tosBOvK4VyD1gdjF","_score":null,"_source":{"@timestamp":"2023-11-09T22:40:28.344713053Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:40:28.344+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=431","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569628344]},{"_index":"continuous-tests-pods-2023.11.09","_id":"agc-tosBOvK4VyD1mSvd","_score":null,"_source":{"@timestamp":"2023-11-09T22:40:31.346147045Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:40:31.345+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=432","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569631346]},{"_index":"continuous-tests-pods-2023.11.09","_id":"awc-tosBOvK4VyD1mSvd","_score":null,"_source":{"@timestamp":"2023-11-09T22:40:34.346783233Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:40:34.346+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=433","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569634346]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Sgc-tosBOvK4VyD1sXz3","_score":null,"_source":{"@timestamp":"2023-11-09T22:40:37.348068052Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:40:37.347+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=434","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569637348]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Swc-tosBOvK4VyD1sXz3","_score":null,"_source":{"@timestamp":"2023-11-09T22:40:40.349405118Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:40:40.349+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=435","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569640349]},{"_index":"continuous-tests-pods-2023.11.09","_id":"twc-tosBOvK4VyD1ytQN","_score":null,"_source":{"@timestamp":"2023-11-09T22:40:43.350745459Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:40:43.350+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=436","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569643350]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uAc-tosBOvK4VyD1ytQN","_score":null,"_source":{"@timestamp":"2023-11-09T22:40:46.351913228Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:40:46.351+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=437","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569646351]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ewg-tosBOvK4VyD14iUm","_score":null,"_source":{"@timestamp":"2023-11-09T22:40:49.353056325Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:40:49.352+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=438","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569649353]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fAg-tosBOvK4VyD14iUm","_score":null,"_source":{"@timestamp":"2023-11-09T22:40:52.353742446Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:40:52.353+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=439","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569652353]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HAg-tosBOvK4VyD19k5E","_score":null,"_source":{"@timestamp":"2023-11-09T22:40:55.354826697Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:40:55.354+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=440","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569655354]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HQg-tosBOvK4VyD19k5E","_score":null,"_source":{"@timestamp":"2023-11-09T22:40:58.355961045Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:40:58.355+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=441","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569658355]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mAg_tosBOvK4VyD1DmtS","_score":null,"_source":{"@timestamp":"2023-11-09T22:41:01.357461751Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:41:01.357+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=442","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569661357]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mQg_tosBOvK4VyD1DmtS","_score":null,"_source":{"@timestamp":"2023-11-09T22:41:04.358248760Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:41:04.357+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=443","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569664358]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GAg_tosBOvK4VyD1Jodp","_score":null,"_source":{"@timestamp":"2023-11-09T22:41:07.359566124Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:41:07.359+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=444","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569667359]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GQg_tosBOvK4VyD1Jodp","_score":null,"_source":{"@timestamp":"2023-11-09T22:41:10.360749027Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:41:10.360+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=445","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569670360]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tQg_tosBOvK4VyD1PqKE","_score":null,"_source":{"@timestamp":"2023-11-09T22:41:13.361952327Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:41:13.361+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=446","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569673361]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tgg_tosBOvK4VyD1PqKE","_score":null,"_source":{"@timestamp":"2023-11-09T22:41:16.363312615Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:41:16.362+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=447","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569676363]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ywg_tosBOvK4VyD1VuGc","_score":null,"_source":{"@timestamp":"2023-11-09T22:41:19.364408473Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:41:19.364+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=448","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569679364]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZAg_tosBOvK4VyD1VuGc","_score":null,"_source":{"@timestamp":"2023-11-09T22:41:22.365621795Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:41:22.365+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=449","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569682365]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Hwk_tosBOvK4VyD1biaw","_score":null,"_source":{"@timestamp":"2023-11-09T22:41:25.367047507Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:41:25.366+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=450","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569685367]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IAk_tosBOvK4VyD1biaw","_score":null,"_source":{"@timestamp":"2023-11-09T22:41:28.368107315Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:41:28.367+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=451","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569688368]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IQk_tosBOvK4VyD1biaw","_score":null,"_source":{"@timestamp":"2023-11-09T22:41:31.369147061Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:41:31.368+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=452","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569691369]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2Qk_tosBOvK4VyD1joLQ","_score":null,"_source":{"@timestamp":"2023-11-09T22:41:34.370525444Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:41:34.370+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=453","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569694370]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2gk_tosBOvK4VyD1joLQ","_score":null,"_source":{"@timestamp":"2023-11-09T22:41:37.371408668Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:41:37.371+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=454","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569697371]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Qwk_tosBOvK4VyD1prbp","_score":null,"_source":{"@timestamp":"2023-11-09T22:41:40.372134176Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:41:40.371+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=455","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569700372]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RAk_tosBOvK4VyD1prbp","_score":null,"_source":{"@timestamp":"2023-11-09T22:41:43.377801065Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:41:43.377+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=456","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569703377]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jwk_tosBOvK4VyD1v-YB","_score":null,"_source":{"@timestamp":"2023-11-09T22:41:46.379014025Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:41:46.378+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=457","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569706379]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kAk_tosBOvK4VyD1v-YB","_score":null,"_source":{"@timestamp":"2023-11-09T22:41:49.379948644Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:41:49.379+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=458","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569709379]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6wo_tosBOvK4VyD11wQZ","_score":null,"_source":{"@timestamp":"2023-11-09T22:41:52.380901630Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:41:52.380+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=459","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569712380]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7Ao_tosBOvK4VyD11wQZ","_score":null,"_source":{"@timestamp":"2023-11-09T22:41:55.382206923Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:41:55.381+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=460","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569715382]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Hgo_tosBOvK4VyD17yIy","_score":null,"_source":{"@timestamp":"2023-11-09T22:41:58.383441968Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:41:58.383+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=461","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569718383]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Hwo_tosBOvK4VyD17yIy","_score":null,"_source":{"@timestamp":"2023-11-09T22:42:01.384163373Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:42:01.383+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=462","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569721384]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0ApAtosBOvK4VyD1B1lL","_score":null,"_source":{"@timestamp":"2023-11-09T22:42:04.389085295Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:42:04.388+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=463","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569724389]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0QpAtosBOvK4VyD1B1lL","_score":null,"_source":{"@timestamp":"2023-11-09T22:42:07.390626078Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:42:07.390+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=464","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569727390]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fApAtosBOvK4VyD1G4dy","_score":null,"_source":{"@timestamp":"2023-11-09T22:42:10.391918854Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:42:10.391+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=465","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569730391]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fQpAtosBOvK4VyD1G4dy","_score":null,"_source":{"@timestamp":"2023-11-09T22:42:13.392475909Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:42:13.392+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=466","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569733392]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hgpAtosBOvK4VyD1M7J4","_score":null,"_source":{"@timestamp":"2023-11-09T22:42:16.394019868Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:42:16.393+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=467","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569736394]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hwpAtosBOvK4VyD1M7J4","_score":null,"_source":{"@timestamp":"2023-11-09T22:42:19.394868009Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:42:19.394+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=468","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569739394]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZQpAtosBOvK4VyD1S8-Q","_score":null,"_source":{"@timestamp":"2023-11-09T22:42:22.396500542Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:42:22.396+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=469","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569742396]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZgpAtosBOvK4VyD1S8-Q","_score":null,"_source":{"@timestamp":"2023-11-09T22:42:25.397705745Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:42:25.397+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=470","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569745397]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wAtAtosBOvK4VyD1Yx6k","_score":null,"_source":{"@timestamp":"2023-11-09T22:42:28.398671967Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:42:28.398+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=471","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569748398]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wQtAtosBOvK4VyD1Yx6k","_score":null,"_source":{"@timestamp":"2023-11-09T22:42:31.404359022Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:42:31.404+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=472","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569751404]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QQtAtosBOvK4VyD1e3fB","_score":null,"_source":{"@timestamp":"2023-11-09T22:42:34.405023095Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:42:34.404+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=473","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569754405]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QgtAtosBOvK4VyD1e3fB","_score":null,"_source":{"@timestamp":"2023-11-09T22:42:37.405877159Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:42:37.405+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=474","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569757405]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9AtAtosBOvK4VyD1k83a","_score":null,"_source":{"@timestamp":"2023-11-09T22:42:40.406758163Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:42:40.406+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=475","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569760406]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9QtAtosBOvK4VyD1k83a","_score":null,"_source":{"@timestamp":"2023-11-09T22:42:43.408252007Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:42:43.407+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=476","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569763408]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OQxAtosBOvK4VyD1pw_w","_score":null,"_source":{"@timestamp":"2023-11-09T22:42:46.409566965Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:42:46.409+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=477","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569766409]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OgxAtosBOvK4VyD1pw_w","_score":null,"_source":{"@timestamp":"2023-11-09T22:42:49.410678439Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:42:49.410+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=478","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569769410]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jgxAtosBOvK4VyD1wGEK","_score":null,"_source":{"@timestamp":"2023-11-09T22:42:52.415938008Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:42:52.415+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=479","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569772415]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jwxAtosBOvK4VyD1wGEK","_score":null,"_source":{"@timestamp":"2023-11-09T22:42:55.416936524Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:42:55.416+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=480","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569775416]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jwxAtosBOvK4VyD12H4i","_score":null,"_source":{"@timestamp":"2023-11-09T22:42:58.417964996Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:42:58.417+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=481","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569778417]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kAxAtosBOvK4VyD12H4i","_score":null,"_source":{"@timestamp":"2023-11-09T22:43:01.419345657Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:43:01.418+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=482","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569781419]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kQxAtosBOvK4VyD12H4i","_score":null,"_source":{"@timestamp":"2023-11-09T22:43:03.857076106Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:43:03.856+00:00 Cleaning up expired records tid=1 + count=483","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569783857]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kgxAtosBOvK4VyD12H4i","_score":null,"_source":{"@timestamp":"2023-11-09T22:43:03.857291307Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:43:03.856+00:00 Cleaned up expired records tid=1 + size=0 count=484","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569783857]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kwxAtosBOvK4VyD12H4i","_score":null,"_source":{"@timestamp":"2023-11-09T22:43:03.857318605Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:43:03.857+00:00 Cleaning up query iterator tid=1 + count=485","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569783857]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lAxAtosBOvK4VyD12H4i","_score":null,"_source":{"@timestamp":"2023-11-09T22:43:03.857325643Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:43:03.857+00:00 Cleaning up orphaned records tid=1 + count=486","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569783857]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lQxAtosBOvK4VyD12H4i","_score":null,"_source":{"@timestamp":"2023-11-09T22:43:03.857420005Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:43:03.857+00:00 Cleaning up query iterator tid=1 + count=487","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569783857]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lgxAtosBOvK4VyD12H4i","_score":null,"_source":{"@timestamp":"2023-11-09T22:43:04.420538304Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:43:04.420+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=488","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569784420]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aQxAtosBOvK4VyD1-qlN","_score":null,"_source":{"@timestamp":"2023-11-09T22:43:07.421539478Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:43:07.421+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=489","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569787421]},{"_index":"continuous-tests-pods-2023.11.09","_id":"agxAtosBOvK4VyD1-qlN","_score":null,"_source":{"@timestamp":"2023-11-09T22:43:10.422275112Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:43:10.421+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=490","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569790422]},{"_index":"continuous-tests-pods-2023.11.09","_id":"swxBtosBOvK4VyD1Eslj","_score":null,"_source":{"@timestamp":"2023-11-09T22:43:13.423545065Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:43:13.423+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=491","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569793423]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tAxBtosBOvK4VyD1Eslj","_score":null,"_source":{"@timestamp":"2023-11-09T22:43:16.424388783Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:43:16.424+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=492","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569796424]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pA1BtosBOvK4VyD1KgV_","_score":null,"_source":{"@timestamp":"2023-11-09T22:43:19.430478984Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:43:19.430+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=493","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569799430]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pQ1BtosBOvK4VyD1KgV_","_score":null,"_source":{"@timestamp":"2023-11-09T22:43:22.431449995Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:43:22.431+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=494","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569802431]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sw1BtosBOvK4VyD1QkWW","_score":null,"_source":{"@timestamp":"2023-11-09T22:43:25.433016321Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:43:25.432+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=495","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569805433]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tA1BtosBOvK4VyD1QkWW","_score":null,"_source":{"@timestamp":"2023-11-09T22:43:28.434205602Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:43:28.433+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=496","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569808434]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Gg1BtosBOvK4VyD1Wpyv","_score":null,"_source":{"@timestamp":"2023-11-09T22:43:31.435795662Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:43:31.435+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=497","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569811435]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Gw1BtosBOvK4VyD1Wpyv","_score":null,"_source":{"@timestamp":"2023-11-09T22:43:34.436598824Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:43:34.436+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=498","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569814436]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zw1BtosBOvK4VyD1csnH","_score":null,"_source":{"@timestamp":"2023-11-09T22:43:37.438195290Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:43:37.437+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=499","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569817438]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0A1BtosBOvK4VyD1csnH","_score":null,"_source":{"@timestamp":"2023-11-09T22:43:40.439448529Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:43:40.439+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=500","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569820439]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Aw1BtosBOvK4VyD1hvvc","_score":null,"_source":{"@timestamp":"2023-11-09T22:43:43.440280680Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:43:43.439+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=501","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569823440]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BA1BtosBOvK4VyD1hvvc","_score":null,"_source":{"@timestamp":"2023-11-09T22:43:46.446104691Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:43:46.445+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=502","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569826446]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6w5BtosBOvK4VyD1nhfw","_score":null,"_source":{"@timestamp":"2023-11-09T22:43:49.446822487Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:43:49.446+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=503","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569829446]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7A5BtosBOvK4VyD1nhfw","_score":null,"_source":{"@timestamp":"2023-11-09T22:43:52.448463028Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:43:52.448+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=504","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569832448]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GQ5BtosBOvK4VyD1tzUH","_score":null,"_source":{"@timestamp":"2023-11-09T22:43:55.449056554Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:43:55.448+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=505","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569835449]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Gg5BtosBOvK4VyD1tzUH","_score":null,"_source":{"@timestamp":"2023-11-09T22:43:58.449934555Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:43:58.449+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=506","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569838449]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pw5BtosBOvK4VyD1z10c","_score":null,"_source":{"@timestamp":"2023-11-09T22:44:01.451480545Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:44:01.451+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=507","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569841451]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qA5BtosBOvK4VyD1z10c","_score":null,"_source":{"@timestamp":"2023-11-09T22:44:04.452847061Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:44:04.452+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=508","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569844452]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ww5BtosBOvK4VyD155Y2","_score":null,"_source":{"@timestamp":"2023-11-09T22:44:07.453416880Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:44:07.453+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=509","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569847453]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xA5BtosBOvK4VyD155Y2","_score":null,"_source":{"@timestamp":"2023-11-09T22:44:10.454084831Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:44:10.453+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=510","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569850454]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kg5BtosBOvK4VyD1_8lO","_score":null,"_source":{"@timestamp":"2023-11-09T22:44:13.454922917Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:44:13.454+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=511","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569853454]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kw5BtosBOvK4VyD1_8lO","_score":null,"_source":{"@timestamp":"2023-11-09T22:44:16.456070207Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:44:16.455+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=512","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569856456]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iQ5CtosBOvK4VyD1E-Nn","_score":null,"_source":{"@timestamp":"2023-11-09T22:44:19.456961441Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:44:19.456+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=513","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569859456]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ig5CtosBOvK4VyD1E-Nn","_score":null,"_source":{"@timestamp":"2023-11-09T22:44:22.457979400Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:44:22.457+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=514","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569862457]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_g9CtosBOvK4VyD1Kx9_","_score":null,"_source":{"@timestamp":"2023-11-09T22:44:25.459455293Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:44:25.459+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=515","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569865459]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_w9CtosBOvK4VyD1Kx9_","_score":null,"_source":{"@timestamp":"2023-11-09T22:44:28.461023786Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:44:28.460+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=516","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569868461]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Rg9CtosBOvK4VyD1Q36V","_score":null,"_source":{"@timestamp":"2023-11-09T22:44:31.461869963Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:44:31.461+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=517","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569871461]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Rw9CtosBOvK4VyD1Q36V","_score":null,"_source":{"@timestamp":"2023-11-09T22:44:34.463087150Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:44:34.462+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=518","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569874463]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WA9CtosBOvK4VyD1W9ay","_score":null,"_source":{"@timestamp":"2023-11-09T22:44:37.464222988Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:44:37.463+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=519","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569877464]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WQ9CtosBOvK4VyD1W9ay","_score":null,"_source":{"@timestamp":"2023-11-09T22:44:40.465076439Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:44:40.464+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=520","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569880465]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bhBCtosBOvK4VyD1czDJ","_score":null,"_source":{"@timestamp":"2023-11-09T22:44:43.466635118Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:44:43.466+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=521","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569883466]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bxBCtosBOvK4VyD1czDJ","_score":null,"_source":{"@timestamp":"2023-11-09T22:44:46.467649453Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:44:46.467+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=522","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569886467]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4xBCtosBOvK4VyD1i33l","_score":null,"_source":{"@timestamp":"2023-11-09T22:44:49.468770435Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:44:49.468+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=523","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569889468]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5BBCtosBOvK4VyD1i33l","_score":null,"_source":{"@timestamp":"2023-11-09T22:44:52.470423131Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:44:52.469+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=524","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569892470]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5RBCtosBOvK4VyD1i33l","_score":null,"_source":{"@timestamp":"2023-11-09T22:44:55.470732943Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:44:55.470+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=525","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569895470]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sxBCtosBOvK4VyD1rKMH","_score":null,"_source":{"@timestamp":"2023-11-09T22:44:58.471575411Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:44:58.471+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=526","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569898471]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tBBCtosBOvK4VyD1rKMH","_score":null,"_source":{"@timestamp":"2023-11-09T22:45:01.473007681Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:45:01.472+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=527","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569901473]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1xBCtosBOvK4VyD1xMAh","_score":null,"_source":{"@timestamp":"2023-11-09T22:45:04.474048316Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:45:04.473+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=528","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569904474]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2BBCtosBOvK4VyD1xMAh","_score":null,"_source":{"@timestamp":"2023-11-09T22:45:07.475460169Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:45:07.475+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=529","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569907475]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SRBCtosBOvK4VyD13N44","_score":null,"_source":{"@timestamp":"2023-11-09T22:45:10.481667429Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:45:10.481+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=530","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569910481]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ShBCtosBOvK4VyD13N44","_score":null,"_source":{"@timestamp":"2023-11-09T22:45:13.483255251Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:45:13.482+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=531","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569913483]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1RFCtosBOvK4VyD19AxN","_score":null,"_source":{"@timestamp":"2023-11-09T22:45:16.484288868Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:45:16.483+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=532","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569916484]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1hFCtosBOvK4VyD19AxN","_score":null,"_source":{"@timestamp":"2023-11-09T22:45:19.485253240Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:45:19.484+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=533","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569919485]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vhFDtosBOvK4VyD1DVJP","_score":null,"_source":{"@timestamp":"2023-11-09T22:45:22.486659775Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:45:22.486+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=534","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569922486]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vxFDtosBOvK4VyD1DVJP","_score":null,"_source":{"@timestamp":"2023-11-09T22:45:25.487613132Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:45:25.487+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=535","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569925487]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fRFDtosBOvK4VyD1IZNl","_score":null,"_source":{"@timestamp":"2023-11-09T22:45:28.488480610Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:45:28.488+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=536","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569928488]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fhFDtosBOvK4VyD1IZNl","_score":null,"_source":{"@timestamp":"2023-11-09T22:45:31.490047175Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:45:31.489+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=537","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569931490]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zhFDtosBOvK4VyD1Odx7","_score":null,"_source":{"@timestamp":"2023-11-09T22:45:34.491316590Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:45:34.491+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=538","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569934491]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zxFDtosBOvK4VyD1Odx7","_score":null,"_source":{"@timestamp":"2023-11-09T22:45:37.491826075Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:45:37.491+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=539","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569937491]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_hJDtosBOvK4VyD1URaS","_score":null,"_source":{"@timestamp":"2023-11-09T22:45:40.493170517Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:45:40.492+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=540","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569940493]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_xJDtosBOvK4VyD1URaS","_score":null,"_source":{"@timestamp":"2023-11-09T22:45:43.494628483Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:45:43.494+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=541","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569943494]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fRJDtosBOvK4VyD1aTSp","_score":null,"_source":{"@timestamp":"2023-11-09T22:45:46.495988536Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:45:46.495+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=542","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569946495]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fhJDtosBOvK4VyD1aTSp","_score":null,"_source":{"@timestamp":"2023-11-09T22:45:49.497249193Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:45:49.496+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=543","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569949497]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VBJDtosBOvK4VyD1gVPF","_score":null,"_source":{"@timestamp":"2023-11-09T22:45:52.498345677Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:45:52.497+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=544","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569952498]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VRJDtosBOvK4VyD1gVPF","_score":null,"_source":{"@timestamp":"2023-11-09T22:45:55.499979713Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:45:55.499+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=545","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569955499]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UhJDtosBOvK4VyD1mXDf","_score":null,"_source":{"@timestamp":"2023-11-09T22:45:58.501512395Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:45:58.501+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=546","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569958501]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UxJDtosBOvK4VyD1mXDf","_score":null,"_source":{"@timestamp":"2023-11-09T22:46:01.502290483Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:46:01.501+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=547","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569961502]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_xJDtosBOvK4VyD1rZ77","_score":null,"_source":{"@timestamp":"2023-11-09T22:46:04.503768800Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:46:04.503+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=548","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569964503]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ABJDtosBOvK4VyD1rZ_7","_score":null,"_source":{"@timestamp":"2023-11-09T22:46:07.504911767Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:46:07.504+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=549","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569967504]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ThJDtosBOvK4VyD1xtkS","_score":null,"_source":{"@timestamp":"2023-11-09T22:46:10.510534478Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:46:10.510+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=550","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569970510]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TxJDtosBOvK4VyD1xtkS","_score":null,"_source":{"@timestamp":"2023-11-09T22:46:13.512331436Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:46:13.511+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=551","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569973512]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tRJDtosBOvK4VyD13v8s","_score":null,"_source":{"@timestamp":"2023-11-09T22:46:16.513297257Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:46:16.512+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=552","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569976513]},{"_index":"continuous-tests-pods-2023.11.09","_id":"thJDtosBOvK4VyD13v8s","_score":null,"_source":{"@timestamp":"2023-11-09T22:46:19.514206133Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:46:19.513+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=553","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569979514]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GRNDtosBOvK4VyD19iNH","_score":null,"_source":{"@timestamp":"2023-11-09T22:46:22.515672976Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:46:22.515+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=554","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569982515]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GhNDtosBOvK4VyD19iNH","_score":null,"_source":{"@timestamp":"2023-11-09T22:46:25.516485210Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:46:25.516+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=555","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569985516]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3RNEtosBOvK4VyD1Dntb","_score":null,"_source":{"@timestamp":"2023-11-09T22:46:28.517840819Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:46:28.517+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=556","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569988517]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3hNEtosBOvK4VyD1Dntb","_score":null,"_source":{"@timestamp":"2023-11-09T22:46:31.519068903Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:46:31.518+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=557","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569991519]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XBNEtosBOvK4VyD1Jtdy","_score":null,"_source":{"@timestamp":"2023-11-09T22:46:34.520133565Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:46:34.519+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=558","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569994520]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XRNEtosBOvK4VyD1Jtdy","_score":null,"_source":{"@timestamp":"2023-11-09T22:46:37.521144425Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:46:37.520+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=559","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699569997521]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nhREtosBOvK4VyD1OiGM","_score":null,"_source":{"@timestamp":"2023-11-09T22:46:40.521883637Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:46:40.521+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=560","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570000521]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nxREtosBOvK4VyD1OiGM","_score":null,"_source":{"@timestamp":"2023-11-09T22:46:43.523200403Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:46:43.522+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=561","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570003523]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GBREtosBOvK4VyD1Unql","_score":null,"_source":{"@timestamp":"2023-11-09T22:46:46.524521978Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:46:46.524+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=562","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570006524]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GRREtosBOvK4VyD1Unql","_score":null,"_source":{"@timestamp":"2023-11-09T22:46:49.525705580Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:46:49.525+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=563","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570009525]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zBREtosBOvK4VyD1arG9","_score":null,"_source":{"@timestamp":"2023-11-09T22:46:52.527194838Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:46:52.526+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=564","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570012527]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zRREtosBOvK4VyD1arG9","_score":null,"_source":{"@timestamp":"2023-11-09T22:46:55.528204642Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:46:55.527+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=565","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570015528]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NxREtosBOvK4VyD1gtDX","_score":null,"_source":{"@timestamp":"2023-11-09T22:46:58.533699304Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:46:58.533+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=566","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570018533]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OBREtosBOvK4VyD1gtDX","_score":null,"_source":{"@timestamp":"2023-11-09T22:47:01.535076694Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:47:01.534+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=567","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570021535]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QBREtosBOvK4VyD1mu37","_score":null,"_source":{"@timestamp":"2023-11-09T22:47:04.535951385Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:47:04.535+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=568","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570024535]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QRREtosBOvK4VyD1mu37","_score":null,"_source":{"@timestamp":"2023-11-09T22:47:07.537523054Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:47:07.537+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=569","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570027537]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OhVEtosBOvK4VyD1swoJ","_score":null,"_source":{"@timestamp":"2023-11-09T22:47:10.538440751Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:47:10.538+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=570","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570030538]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OxVEtosBOvK4VyD1swoJ","_score":null,"_source":{"@timestamp":"2023-11-09T22:47:13.539226878Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:47:13.538+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=571","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570033539]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CBVEtosBOvK4VyD1xycf","_score":null,"_source":{"@timestamp":"2023-11-09T22:47:16.539988740Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:47:16.539+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=572","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570036539]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CRVEtosBOvK4VyD1xycf","_score":null,"_source":{"@timestamp":"2023-11-09T22:47:19.541099439Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:47:19.540+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=573","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570039541]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qhVEtosBOvK4VyD132E2","_score":null,"_source":{"@timestamp":"2023-11-09T22:47:22.541775696Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:47:22.541+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=574","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570042541]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qxVEtosBOvK4VyD132E2","_score":null,"_source":{"@timestamp":"2023-11-09T22:47:25.542962712Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:47:25.542+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=575","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570045542]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0BVEtosBOvK4VyD196RR","_score":null,"_source":{"@timestamp":"2023-11-09T22:47:28.544048911Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:47:28.543+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=576","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570048544]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0RVEtosBOvK4VyD196RR","_score":null,"_source":{"@timestamp":"2023-11-09T22:47:31.545134273Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:47:31.544+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=577","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570051545]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hhVFtosBOvK4VyD1D_lh","_score":null,"_source":{"@timestamp":"2023-11-09T22:47:34.546737434Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:47:34.546+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=578","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570054546]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hxVFtosBOvK4VyD1D_lh","_score":null,"_source":{"@timestamp":"2023-11-09T22:47:37.548199082Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:47:37.547+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=579","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570057548]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BRZFtosBOvK4VyD1Jyd3","_score":null,"_source":{"@timestamp":"2023-11-09T22:47:40.549758197Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:47:40.549+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=580","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570060549]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BhZFtosBOvK4VyD1Jyd3","_score":null,"_source":{"@timestamp":"2023-11-09T22:47:43.550244635Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:47:43.549+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=581","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570063550]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EBZFtosBOvK4VyD1P1yS","_score":null,"_source":{"@timestamp":"2023-11-09T22:47:46.551662135Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:47:46.551+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=582","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570066551]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ERZFtosBOvK4VyD1P1yS","_score":null,"_source":{"@timestamp":"2023-11-09T22:47:49.552918413Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:47:49.552+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=583","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570069552]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EhZFtosBOvK4VyD1P1yS","_score":null,"_source":{"@timestamp":"2023-11-09T22:47:52.558106913Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:47:52.557+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=584","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570072558]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wRZFtosBOvK4VyD1X4O4","_score":null,"_source":{"@timestamp":"2023-11-09T22:47:55.558985996Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:47:55.558+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=585","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570075558]},{"_index":"continuous-tests-pods-2023.11.09","_id":"whZFtosBOvK4VyD1X4O4","_score":null,"_source":{"@timestamp":"2023-11-09T22:47:58.560396445Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:47:58.560+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=586","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570078560]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GhZFtosBOvK4VyD1d6fP","_score":null,"_source":{"@timestamp":"2023-11-09T22:48:01.561164611Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:48:01.560+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=587","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570081561]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GxZFtosBOvK4VyD1d6fP","_score":null,"_source":{"@timestamp":"2023-11-09T22:48:03.862827083Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:48:03.862+00:00 Cleaning up expired records tid=1 + count=588","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570083862]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HBZFtosBOvK4VyD1d6fP","_score":null,"_source":{"@timestamp":"2023-11-09T22:48:03.862929761Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:48:03.862+00:00 Cleaned up expired records tid=1 + size=0 count=589","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570083862]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HRZFtosBOvK4VyD1d6fP","_score":null,"_source":{"@timestamp":"2023-11-09T22:48:03.862937560Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:48:03.862+00:00 Cleaning up query iterator tid=1 + count=590","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570083862]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HhZFtosBOvK4VyD1d6fP","_score":null,"_source":{"@timestamp":"2023-11-09T22:48:03.862952435Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:48:03.862+00:00 Cleaning up orphaned records tid=1 + count=591","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570083862]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HxZFtosBOvK4VyD1d6fP","_score":null,"_source":{"@timestamp":"2023-11-09T22:48:03.862956763Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:48:03.862+00:00 Cleaning up query iterator tid=1 + count=592","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570083862]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IBZFtosBOvK4VyD1d6fP","_score":null,"_source":{"@timestamp":"2023-11-09T22:48:04.562204717Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:48:04.561+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=593","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570084562]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aBZFtosBOvK4VyD1j-Du","_score":null,"_source":{"@timestamp":"2023-11-09T22:48:07.563058363Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:48:07.562+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=594","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570087563]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aRZFtosBOvK4VyD1j-Du","_score":null,"_source":{"@timestamp":"2023-11-09T22:48:10.564765889Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:48:10.564+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=595","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570090564]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kRdFtosBOvK4VyD1qBYK","_score":null,"_source":{"@timestamp":"2023-11-09T22:48:13.566465741Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:48:13.565+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=596","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570093566]},{"_index":"continuous-tests-pods-2023.11.09","_id":"khdFtosBOvK4VyD1qBYK","_score":null,"_source":{"@timestamp":"2023-11-09T22:48:16.568111904Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:48:16.567+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=597","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570096568]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UxdFtosBOvK4VyD1wDMh","_score":null,"_source":{"@timestamp":"2023-11-09T22:48:19.569114699Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:48:19.568+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=598","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570099569]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VBdFtosBOvK4VyD1wDMh","_score":null,"_source":{"@timestamp":"2023-11-09T22:48:22.570003001Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:48:22.569+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=599","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570102570]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hRdFtosBOvK4VyD12Hc5","_score":null,"_source":{"@timestamp":"2023-11-09T22:48:25.571626565Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:48:25.571+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=600","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570105571]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hhdFtosBOvK4VyD12Hc5","_score":null,"_source":{"@timestamp":"2023-11-09T22:48:28.572230556Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:48:28.571+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=601","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570108572]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LBdFtosBOvK4VyD17LxT","_score":null,"_source":{"@timestamp":"2023-11-09T22:48:31.573772222Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:48:31.573+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=602","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570111573]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LRdFtosBOvK4VyD17LxT","_score":null,"_source":{"@timestamp":"2023-11-09T22:48:34.574741117Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:48:34.574+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=603","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570114574]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ThhGtosBOvK4VyD1BBFo","_score":null,"_source":{"@timestamp":"2023-11-09T22:48:37.575103623Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:48:37.574+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=604","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570117575]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TxhGtosBOvK4VyD1BBFo","_score":null,"_source":{"@timestamp":"2023-11-09T22:48:40.576521251Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:48:40.576+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=605","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570120576]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZRhGtosBOvK4VyD1HGl_","_score":null,"_source":{"@timestamp":"2023-11-09T22:48:43.577420976Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:48:43.577+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=606","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570123577]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZhhGtosBOvK4VyD1HGl_","_score":null,"_source":{"@timestamp":"2023-11-09T22:48:46.578799104Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:48:46.578+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=607","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570126578]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BBhGtosBOvK4VyD1NMCm","_score":null,"_source":{"@timestamp":"2023-11-09T22:48:49.579957738Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:48:49.579+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=608","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570129579]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BRhGtosBOvK4VyD1NMCm","_score":null,"_source":{"@timestamp":"2023-11-09T22:48:52.580710317Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:48:52.580+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=609","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570132580]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TRhGtosBOvK4VyD1TOPB","_score":null,"_source":{"@timestamp":"2023-11-09T22:48:55.582157516Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:48:55.581+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=610","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570135582]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ThhGtosBOvK4VyD1TOPB","_score":null,"_source":{"@timestamp":"2023-11-09T22:48:58.583286172Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:48:58.582+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=611","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570138583]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bxlGtosBOvK4VyD1ZADL","_score":null,"_source":{"@timestamp":"2023-11-09T22:49:01.589291681Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:49:01.588+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=612","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570141589]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cBlGtosBOvK4VyD1ZADL","_score":null,"_source":{"@timestamp":"2023-11-09T22:49:04.590698417Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:49:04.590+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=613","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570144590]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cRlGtosBOvK4VyD1ZADL","_score":null,"_source":{"@timestamp":"2023-11-09T22:49:07.591766461Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:49:07.591+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=614","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570147591]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9BlGtosBOvK4VyD1hCn4","_score":null,"_source":{"@timestamp":"2023-11-09T22:49:10.593223903Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:49:10.592+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=615","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570150593]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9RlGtosBOvK4VyD1hCn4","_score":null,"_source":{"@timestamp":"2023-11-09T22:49:13.594898573Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:49:13.594+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=616","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570153594]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3xlGtosBOvK4VyD1nUgR","_score":null,"_source":{"@timestamp":"2023-11-09T22:49:16.595999376Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:49:16.595+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=617","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570156595]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4BlGtosBOvK4VyD1nUgR","_score":null,"_source":{"@timestamp":"2023-11-09T22:49:19.597967040Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:49:19.597+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=618","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570159597]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9xlGtosBOvK4VyD1tZIo","_score":null,"_source":{"@timestamp":"2023-11-09T22:49:22.598467607Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:49:22.598+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=619","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570162598]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-BlGtosBOvK4VyD1tZIo","_score":null,"_source":{"@timestamp":"2023-11-09T22:49:25.599364977Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:49:25.599+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=620","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570165599]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_xlGtosBOvK4VyD1zdw_","_score":null,"_source":{"@timestamp":"2023-11-09T22:49:28.600421856Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:49:28.600+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=621","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570168600]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AhlGtosBOvK4VyD1zd0_","_score":null,"_source":{"@timestamp":"2023-11-09T22:49:31.601533002Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:49:31.600+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=622","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570171601]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TBpGtosBOvK4VyD15S9Y","_score":null,"_source":{"@timestamp":"2023-11-09T22:49:34.602547604Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:49:34.602+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=623","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570174602]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TRpGtosBOvK4VyD15S9Y","_score":null,"_source":{"@timestamp":"2023-11-09T22:49:37.604250194Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:49:37.603+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=624","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570177604]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2RpGtosBOvK4VyD1_WZ1","_score":null,"_source":{"@timestamp":"2023-11-09T22:49:40.604976180Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:49:40.604+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=625","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570180604]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2hpGtosBOvK4VyD1_WZ1","_score":null,"_source":{"@timestamp":"2023-11-09T22:49:43.606301237Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:49:43.605+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=626","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570183606]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5hpHtosBOvK4VyD1EX-L","_score":null,"_source":{"@timestamp":"2023-11-09T22:49:46.607616376Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:49:46.607+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=627","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570186607]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5xpHtosBOvK4VyD1EX-L","_score":null,"_source":{"@timestamp":"2023-11-09T22:49:49.608923622Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:49:49.608+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=628","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570189608]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TxpHtosBOvK4VyD1KZ6d","_score":null,"_source":{"@timestamp":"2023-11-09T22:49:52.610062561Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:49:52.609+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=629","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570192610]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UBpHtosBOvK4VyD1KZ6d","_score":null,"_source":{"@timestamp":"2023-11-09T22:49:55.611698040Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:49:55.611+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=630","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570195611]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1xpHtosBOvK4VyD1Qbm2","_score":null,"_source":{"@timestamp":"2023-11-09T22:49:58.613069797Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:49:58.612+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=631","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570198613]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2BpHtosBOvK4VyD1Qbm2","_score":null,"_source":{"@timestamp":"2023-11-09T22:50:01.614256491Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:50:01.613+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=632","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570201614]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1RpHtosBOvK4VyD1WenO","_score":null,"_source":{"@timestamp":"2023-11-09T22:50:04.615744107Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:50:04.615+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=633","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570204615]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1hpHtosBOvK4VyD1WenO","_score":null,"_source":{"@timestamp":"2023-11-09T22:50:07.617635448Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:50:07.617+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=634","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570207617]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aBtHtosBOvK4VyD1cSTu","_score":null,"_source":{"@timestamp":"2023-11-09T22:50:10.618973175Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:50:10.618+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=635","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570210618]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aRtHtosBOvK4VyD1cSTu","_score":null,"_source":{"@timestamp":"2023-11-09T22:50:13.619835598Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:50:13.619+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=636","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570213619]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dxtHtosBOvK4VyD1ikwL","_score":null,"_source":{"@timestamp":"2023-11-09T22:50:16.620445196Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:50:16.620+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=637","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570216620]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eBtHtosBOvK4VyD1ikwL","_score":null,"_source":{"@timestamp":"2023-11-09T22:50:19.625777415Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:50:19.625+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=638","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570219625]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cxtHtosBOvK4VyD1nmYh","_score":null,"_source":{"@timestamp":"2023-11-09T22:50:22.626853671Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:50:22.626+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=639","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570222626]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dBtHtosBOvK4VyD1nmYh","_score":null,"_source":{"@timestamp":"2023-11-09T22:50:25.628186138Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:50:25.627+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=640","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570225628]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RxtHtosBOvK4VyD1tq86","_score":null,"_source":{"@timestamp":"2023-11-09T22:50:28.629334058Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:50:28.628+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=641","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570228629]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SBtHtosBOvK4VyD1tq86","_score":null,"_source":{"@timestamp":"2023-11-09T22:50:31.631214419Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:50:31.630+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=642","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570231631]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4xxHtosBOvK4VyD1zgFQ","_score":null,"_source":{"@timestamp":"2023-11-09T22:50:34.632128302Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:50:34.631+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=643","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570234632]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5BxHtosBOvK4VyD1zgFQ","_score":null,"_source":{"@timestamp":"2023-11-09T22:50:37.633802076Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:50:37.633+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=644","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570237633]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YhxHtosBOvK4VyD15lhp","_score":null,"_source":{"@timestamp":"2023-11-09T22:50:40.634609282Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:50:40.634+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=645","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570240634]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YxxHtosBOvK4VyD15lhp","_score":null,"_source":{"@timestamp":"2023-11-09T22:50:43.636050906Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:50:43.635+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=646","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570243636]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DBxHtosBOvK4VyD1_q-A","_score":null,"_source":{"@timestamp":"2023-11-09T22:50:46.637174360Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:50:46.636+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=647","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570246637]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DRxHtosBOvK4VyD1_q-A","_score":null,"_source":{"@timestamp":"2023-11-09T22:50:49.640796726Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:50:49.637+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=648","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570249640]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rBxItosBOvK4VyD1Fv-W","_score":null,"_source":{"@timestamp":"2023-11-09T22:50:52.643859894Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:50:52.643+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=649","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570252643]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rRxItosBOvK4VyD1Fv-W","_score":null,"_source":{"@timestamp":"2023-11-09T22:50:55.645180274Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:50:55.644+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=650","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570255645]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rhxItosBOvK4VyD1Fv-W","_score":null,"_source":{"@timestamp":"2023-11-09T22:50:58.645989157Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:50:58.645+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=651","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570258645]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Dx1ItosBOvK4VyD1Nia5","_score":null,"_source":{"@timestamp":"2023-11-09T22:51:01.648073261Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:51:01.647+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=652","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570261648]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EB1ItosBOvK4VyD1Nia5","_score":null,"_source":{"@timestamp":"2023-11-09T22:51:04.649391701Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:51:04.648+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=653","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570264649]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KR1ItosBOvK4VyD1TkXU","_score":null,"_source":{"@timestamp":"2023-11-09T22:51:07.650540482Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:51:07.650+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=654","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570267650]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Kh1ItosBOvK4VyD1TkXU","_score":null,"_source":{"@timestamp":"2023-11-09T22:51:10.651676723Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:51:10.651+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=655","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570270651]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7B1ItosBOvK4VyD1ZmHv","_score":null,"_source":{"@timestamp":"2023-11-09T22:51:13.652981448Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:51:13.652+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=656","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570273652]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7R1ItosBOvK4VyD1ZmHv","_score":null,"_source":{"@timestamp":"2023-11-09T22:51:16.654587662Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:51:16.654+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=657","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570276654]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lB1ItosBOvK4VyD1f5MF","_score":null,"_source":{"@timestamp":"2023-11-09T22:51:19.656162312Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:51:19.655+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=658","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570279656]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lR1ItosBOvK4VyD1f5MF","_score":null,"_source":{"@timestamp":"2023-11-09T22:51:22.657369735Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:51:22.657+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=659","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570282657]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Lh1ItosBOvK4VyD1l9gh","_score":null,"_source":{"@timestamp":"2023-11-09T22:51:25.658358787Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:51:25.657+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=660","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570285658]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Lx1ItosBOvK4VyD1l9gh","_score":null,"_source":{"@timestamp":"2023-11-09T22:51:28.660088137Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:51:28.659+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=661","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570288660]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Gx5ItosBOvK4VyD1ry04","_score":null,"_source":{"@timestamp":"2023-11-09T22:51:31.665618214Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:51:31.665+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=662","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570291665]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HB5ItosBOvK4VyD1ry04","_score":null,"_source":{"@timestamp":"2023-11-09T22:51:34.667279649Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:51:34.666+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=663","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570294667]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1h5ItosBOvK4VyD1w19M","_score":null,"_source":{"@timestamp":"2023-11-09T22:51:37.668841239Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:51:37.668+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=664","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570297668]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1x5ItosBOvK4VyD1w19M","_score":null,"_source":{"@timestamp":"2023-11-09T22:51:40.669782135Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:51:40.669+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=665","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570300669]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pR5ItosBOvK4VyD125hk","_score":null,"_source":{"@timestamp":"2023-11-09T22:51:43.671205029Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:51:43.670+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=666","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570303671]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ph5ItosBOvK4VyD125hk","_score":null,"_source":{"@timestamp":"2023-11-09T22:51:46.672971804Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:51:46.672+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=667","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570306672]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7B5ItosBOvK4VyD187V9","_score":null,"_source":{"@timestamp":"2023-11-09T22:51:49.674278979Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:51:49.673+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=668","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570309674]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7R5ItosBOvK4VyD187V9","_score":null,"_source":{"@timestamp":"2023-11-09T22:51:52.674935788Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:51:52.674+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=669","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570312674]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5R5JtosBOvK4VyD1C9KW","_score":null,"_source":{"@timestamp":"2023-11-09T22:51:55.676452727Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:51:55.676+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=670","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570315676]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5h5JtosBOvK4VyD1C9KW","_score":null,"_source":{"@timestamp":"2023-11-09T22:51:58.681546608Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:51:58.681+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=671","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570318681]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ox5JtosBOvK4VyD1I_iu","_score":null,"_source":{"@timestamp":"2023-11-09T22:52:01.683000778Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:52:01.682+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=672","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570321683]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PB5JtosBOvK4VyD1I_iu","_score":null,"_source":{"@timestamp":"2023-11-09T22:52:04.684290222Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:52:04.683+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=673","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570324684]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Kx9JtosBOvK4VyD1OzPG","_score":null,"_source":{"@timestamp":"2023-11-09T22:52:07.685579680Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:52:07.685+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=674","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570327685]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LB9JtosBOvK4VyD1OzPG","_score":null,"_source":{"@timestamp":"2023-11-09T22:52:10.687111495Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:52:10.686+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=675","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570330687]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hh9JtosBOvK4VyD1T2Hd","_score":null,"_source":{"@timestamp":"2023-11-09T22:52:13.688804421Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:52:13.688+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=676","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570333688]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hx9JtosBOvK4VyD1T2Hd","_score":null,"_source":{"@timestamp":"2023-11-09T22:52:16.689558244Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:52:16.689+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=677","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570336689]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Wx9JtosBOvK4VyD1Z371","_score":null,"_source":{"@timestamp":"2023-11-09T22:52:19.690833023Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:52:19.690+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=678","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570339690]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XB9JtosBOvK4VyD1Z371","_score":null,"_source":{"@timestamp":"2023-11-09T22:52:22.692383626Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:52:22.691+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=679","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570342692]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GB9JtosBOvK4VyD1gLkM","_score":null,"_source":{"@timestamp":"2023-11-09T22:52:25.697897017Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:52:25.697+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=680","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570345697]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GR9JtosBOvK4VyD1gLkM","_score":null,"_source":{"@timestamp":"2023-11-09T22:52:28.698862960Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:52:28.698+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=681","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570348698]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OyBJtosBOvK4VyD1mBMk","_score":null,"_source":{"@timestamp":"2023-11-09T22:52:31.700069676Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:52:31.699+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=682","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570351700]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PCBJtosBOvK4VyD1mBMk","_score":null,"_source":{"@timestamp":"2023-11-09T22:52:34.701955392Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:52:34.701+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=683","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570354701]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eCBJtosBOvK4VyD1sG47","_score":null,"_source":{"@timestamp":"2023-11-09T22:52:37.703720048Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:52:37.703+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=684","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570357703]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eSBJtosBOvK4VyD1sG47","_score":null,"_source":{"@timestamp":"2023-11-09T22:52:40.704644481Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:52:40.704+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=685","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570360704]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NCBJtosBOvK4VyD1yMhX","_score":null,"_source":{"@timestamp":"2023-11-09T22:52:43.706563192Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:52:43.706+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=686","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570363706]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NSBJtosBOvK4VyD1yMhX","_score":null,"_source":{"@timestamp":"2023-11-09T22:52:46.707807865Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:52:46.707+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=687","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570366707]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NiBJtosBOvK4VyD1yMhX","_score":null,"_source":{"@timestamp":"2023-11-09T22:52:49.709548981Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:52:49.709+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=688","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570369709]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lCFJtosBOvK4VyD16CN3","_score":null,"_source":{"@timestamp":"2023-11-09T22:52:52.711378890Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:52:52.711+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=689","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570372711]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lSFJtosBOvK4VyD16CN3","_score":null,"_source":{"@timestamp":"2023-11-09T22:52:55.712583755Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:52:55.712+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=690","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570375712]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JSFKtosBOvK4VyD1AD-Q","_score":null,"_source":{"@timestamp":"2023-11-09T22:52:58.713742967Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:52:58.713+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=691","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570378713]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JiFKtosBOvK4VyD1AD-Q","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:01.714848299Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:01.714+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=692","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570381714]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JyFKtosBOvK4VyD1AD-Q","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:03.853437279Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:03.853+00:00 Begin iterating blocks... topics=\"codex + discoveryengine\" tid=1 count=693","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570383853]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KCFKtosBOvK4VyD1AD-Q","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:03.853482327Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:03.853+00:00 Iterating blocks finished. topics=\"codex + discoveryengine\" tid=1 count=694","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570383853]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KSFKtosBOvK4VyD1AD-Q","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:03.853488607Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:03.853+00:00 About to sleep advertise loop topics=\"codex + discoveryengine\" tid=1 sleep=30m count=695","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570383853]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KiFKtosBOvK4VyD1AD-Q","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:03.863851425Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:03.863+00:00 Cleaning up expired records tid=1 + count=696","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570383863]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KyFKtosBOvK4VyD1AD-Q","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:03.864096250Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:03.863+00:00 Cleaned up expired records tid=1 + size=0 count=697","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570383864]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LCFKtosBOvK4VyD1AD-Q","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:03.864110047Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:03.863+00:00 Cleaning up query iterator tid=1 + count=698","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570383864]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LSFKtosBOvK4VyD1AD-Q","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:03.864115173Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:03.863+00:00 Cleaning up orphaned records tid=1 + count=699","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570383864]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LiFKtosBOvK4VyD1AD-Q","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:03.864292003Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:03.864+00:00 Cleaning up query iterator tid=1 + count=700","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570383864]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZCFKtosBOvK4VyD1FFex","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:04.716028356Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:04.715+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=701","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570384716]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZSFKtosBOvK4VyD1FFex","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:07.716622140Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:07.716+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=702","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570387716]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GCFKtosBOvK4VyD1LHbG","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:10.717705840Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:10.717+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=703","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570390717]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GSFKtosBOvK4VyD1LHbG","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:13.723812071Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:13.723+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=704","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570393723]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PSFKtosBOvK4VyD1RJPd","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:16.724848976Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:16.724+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=705","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570396724]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PiFKtosBOvK4VyD1RJPd","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:19.726016289Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:19.725+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=706","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570399726]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-CFKtosBOvK4VyD1XNHz","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:22.727088203Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:22.726+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=707","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570402727]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-SFKtosBOvK4VyD1XNHz","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:25.728491659Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:25.728+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=708","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570405728]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kSJKtosBOvK4VyD1dRwH","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:28.730007515Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:28.729+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=709","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570408730]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kiJKtosBOvK4VyD1dRwH","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:31.731249584Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:31.730+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=710","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570411731]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tiJKtosBOvK4VyD1jWsh","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:34.733133449Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:34.732+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=711","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570414733]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tyJKtosBOvK4VyD1jWsh","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:37.734275461Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:37.733+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=712","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570417734]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tyJKtosBOvK4VyD1pac4","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:40.736273217Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:40.735+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=713","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570420736]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uCJKtosBOvK4VyD1pac4","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:43.737131048Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:43.736+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=714","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570423737]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CiJKtosBOvK4VyD1vc5R","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:46.738176020Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:46.737+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=715","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570426738]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CyJKtosBOvK4VyD1vc5R","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:49.739246172Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:49.738+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=716","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570429739]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ySJKtosBOvK4VyD11epq","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:52.744928618Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:52.744+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=717","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570432744]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yiJKtosBOvK4VyD11epq","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:55.745708038Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:55.745+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=718","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570435745]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vyNKtosBOvK4VyD17QeB","_score":null,"_source":{"@timestamp":"2023-11-09T22:53:58.747340594Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:53:58.746+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=719","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570438747]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wCNKtosBOvK4VyD17QeB","_score":null,"_source":{"@timestamp":"2023-11-09T22:54:01.748907234Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:54:01.748+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=720","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570441748]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wSNKtosBOvK4VyD17QeB","_score":null,"_source":{"@timestamp":"2023-11-09T22:54:04.750109672Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:54:04.749+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=721","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570444750]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UiNLtosBOvK4VyD1DVWm","_score":null,"_source":{"@timestamp":"2023-11-09T22:54:07.751926934Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:54:07.751+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=722","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570447751]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UyNLtosBOvK4VyD1DVWm","_score":null,"_source":{"@timestamp":"2023-11-09T22:54:10.753183476Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:54:10.752+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=723","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570450753]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pSNLtosBOvK4VyD1JYm6","_score":null,"_source":{"@timestamp":"2023-11-09T22:54:13.754722144Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:54:13.754+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=724","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570453754]},{"_index":"continuous-tests-pods-2023.11.09","_id":"piNLtosBOvK4VyD1JYm6","_score":null,"_source":{"@timestamp":"2023-11-09T22:54:16.755742043Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:54:16.755+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=725","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570456755]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vCNLtosBOvK4VyD1PabR","_score":null,"_source":{"@timestamp":"2023-11-09T22:54:19.761093806Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:54:19.760+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=726","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570459761]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vSNLtosBOvK4VyD1PabR","_score":null,"_source":{"@timestamp":"2023-11-09T22:54:22.761494717Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:54:22.761+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=727","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570462761]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1CNLtosBOvK4VyD1Vejm","_score":null,"_source":{"@timestamp":"2023-11-09T22:54:25.762670998Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:54:25.762+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=728","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570465762]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1SNLtosBOvK4VyD1Vejm","_score":null,"_source":{"@timestamp":"2023-11-09T22:54:28.764112705Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:54:28.763+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=729","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570468764]},{"_index":"continuous-tests-pods-2023.11.09","_id":"syRLtosBOvK4VyD1bUL9","_score":null,"_source":{"@timestamp":"2023-11-09T22:54:31.765606354Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:54:31.765+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=730","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570471765]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tCRLtosBOvK4VyD1bUL9","_score":null,"_source":{"@timestamp":"2023-11-09T22:54:34.767293321Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:54:34.766+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=731","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570474767]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ICRLtosBOvK4VyD1hp4Z","_score":null,"_source":{"@timestamp":"2023-11-09T22:54:37.769080023Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:54:37.768+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=732","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570477769]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ISRLtosBOvK4VyD1hp4Z","_score":null,"_source":{"@timestamp":"2023-11-09T22:54:40.770771527Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:54:40.770+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=733","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570480770]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HyRLtosBOvK4VyD1muIu","_score":null,"_source":{"@timestamp":"2023-11-09T22:54:43.772530975Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:54:43.772+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=734","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570483772]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ICRLtosBOvK4VyD1muIu","_score":null,"_source":{"@timestamp":"2023-11-09T22:54:46.774481978Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:54:46.774+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=735","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570486774]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WCVLtosBOvK4VyD1sjpG","_score":null,"_source":{"@timestamp":"2023-11-09T22:54:49.775678376Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:54:49.775+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=736","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570489775]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WSVLtosBOvK4VyD1sjpG","_score":null,"_source":{"@timestamp":"2023-11-09T22:54:52.776702645Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:54:52.776+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=737","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570492776]},{"_index":"continuous-tests-pods-2023.11.09","_id":"riVLtosBOvK4VyD1ylln","_score":null,"_source":{"@timestamp":"2023-11-09T22:54:55.778678504Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:54:55.778+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=738","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570495778]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ryVLtosBOvK4VyD1ylln","_score":null,"_source":{"@timestamp":"2023-11-09T22:54:58.779930022Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:54:58.779+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=739","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570498779]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ySVLtosBOvK4VyD14nZ2","_score":null,"_source":{"@timestamp":"2023-11-09T22:55:01.781688026Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:55:01.781+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=740","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570501781]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yiVLtosBOvK4VyD14nZ2","_score":null,"_source":{"@timestamp":"2023-11-09T22:55:04.782090898Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:55:04.781+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=741","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570504782]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ICVLtosBOvK4VyD1-pWQ","_score":null,"_source":{"@timestamp":"2023-11-09T22:55:07.783457058Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:55:07.782+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=742","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570507783]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ISVLtosBOvK4VyD1-pWQ","_score":null,"_source":{"@timestamp":"2023-11-09T22:55:10.784925788Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:55:10.784+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=743","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570510784]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GSVMtosBOvK4VyD1ErKn","_score":null,"_source":{"@timestamp":"2023-11-09T22:55:13.786481390Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:55:13.786+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=744","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570513786]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GiVMtosBOvK4VyD1ErKn","_score":null,"_source":{"@timestamp":"2023-11-09T22:55:16.788047080Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:55:16.787+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=745","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570516788]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GyVMtosBOvK4VyD1ErKn","_score":null,"_source":{"@timestamp":"2023-11-09T22:55:19.789553934Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:55:19.789+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=746","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570519789]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uyVMtosBOvK4VyD1Mv7N","_score":null,"_source":{"@timestamp":"2023-11-09T22:55:22.790566430Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:55:22.790+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=747","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570522790]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vCVMtosBOvK4VyD1Mv7N","_score":null,"_source":{"@timestamp":"2023-11-09T22:55:25.792022894Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:55:25.791+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=748","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570525792]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KSZMtosBOvK4VyD1Skbj","_score":null,"_source":{"@timestamp":"2023-11-09T22:55:28.793770415Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:55:28.793+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=749","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570528793]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KiZMtosBOvK4VyD1Skbj","_score":null,"_source":{"@timestamp":"2023-11-09T22:55:31.795394276Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:55:31.795+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=750","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570531795]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AiZMtosBOvK4VyD1Ypj7","_score":null,"_source":{"@timestamp":"2023-11-09T22:55:34.796852497Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:55:34.796+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=751","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570534796]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AyZMtosBOvK4VyD1Ypj7","_score":null,"_source":{"@timestamp":"2023-11-09T22:55:37.798781416Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:55:37.798+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=752","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570537798]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LiZMtosBOvK4VyD1e9QV","_score":null,"_source":{"@timestamp":"2023-11-09T22:55:40.799861650Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:55:40.799+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=753","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570540799]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LyZMtosBOvK4VyD1e9QV","_score":null,"_source":{"@timestamp":"2023-11-09T22:55:43.801231492Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:55:43.800+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=754","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570543801]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XiZMtosBOvK4VyD1k_Qq","_score":null,"_source":{"@timestamp":"2023-11-09T22:55:46.802573660Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:55:46.802+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=755","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570546802]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XyZMtosBOvK4VyD1k_Qq","_score":null,"_source":{"@timestamp":"2023-11-09T22:55:49.803964134Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:55:49.803+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=756","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570549803]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kSdMtosBOvK4VyD1qxJD","_score":null,"_source":{"@timestamp":"2023-11-09T22:55:52.804554609Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:55:52.804+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=757","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570552804]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kidMtosBOvK4VyD1qxJD","_score":null,"_source":{"@timestamp":"2023-11-09T22:55:55.806005156Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:55:55.805+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=758","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570555806]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UydMtosBOvK4VyD1vyte","_score":null,"_source":{"@timestamp":"2023-11-09T22:55:58.807082090Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:55:58.806+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=759","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570558807]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VCdMtosBOvK4VyD1vyte","_score":null,"_source":{"@timestamp":"2023-11-09T22:56:01.808468163Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:56:01.808+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=760","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570561808]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aSdMtosBOvK4VyD111x4","_score":null,"_source":{"@timestamp":"2023-11-09T22:56:04.809990913Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:56:04.809+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=761","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570564809]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aidMtosBOvK4VyD111x4","_score":null,"_source":{"@timestamp":"2023-11-09T22:56:07.811424117Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:56:07.811+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=762","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570567811]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DidMtosBOvK4VyD175eQ","_score":null,"_source":{"@timestamp":"2023-11-09T22:56:10.813358422Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:56:10.812+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=763","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570570813]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DydMtosBOvK4VyD175eQ","_score":null,"_source":{"@timestamp":"2023-11-09T22:56:13.814922491Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:56:13.814+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=764","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570573814]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fCdNtosBOvK4VyD1B8Gn","_score":null,"_source":{"@timestamp":"2023-11-09T22:56:16.815118608Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:56:16.814+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=765","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570576815]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fSdNtosBOvK4VyD1B8Gn","_score":null,"_source":{"@timestamp":"2023-11-09T22:56:19.817317063Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:56:19.816+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=766","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570579817]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YidNtosBOvK4VyD1H969","_score":null,"_source":{"@timestamp":"2023-11-09T22:56:22.819070651Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:56:22.818+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=767","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570582819]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YydNtosBOvK4VyD1H969","_score":null,"_source":{"@timestamp":"2023-11-09T22:56:25.821154944Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:56:25.820+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=768","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570585821]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fShNtosBOvK4VyD1NzTZ","_score":null,"_source":{"@timestamp":"2023-11-09T22:56:28.823210488Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:56:28.822+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=769","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570588823]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fihNtosBOvK4VyD1NzTZ","_score":null,"_source":{"@timestamp":"2023-11-09T22:56:31.829030457Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:56:31.828+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=770","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570591829]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yihNtosBOvK4VyD1S3vy","_score":null,"_source":{"@timestamp":"2023-11-09T22:56:34.830752947Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:56:34.830+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=771","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570594830]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yyhNtosBOvK4VyD1S3vy","_score":null,"_source":{"@timestamp":"2023-11-09T22:56:37.831944256Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:56:37.831+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=772","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570597831]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PihNtosBOvK4VyD1ZNQJ","_score":null,"_source":{"@timestamp":"2023-11-09T22:56:40.833176618Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:56:40.832+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=773","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570600833]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PyhNtosBOvK4VyD1ZNQJ","_score":null,"_source":{"@timestamp":"2023-11-09T22:56:43.833742165Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:56:43.833+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=774","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570603833]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HClNtosBOvK4VyD1fCsi","_score":null,"_source":{"@timestamp":"2023-11-09T22:56:46.835752308Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:56:46.835+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=775","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570606835]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HSlNtosBOvK4VyD1fCsi","_score":null,"_source":{"@timestamp":"2023-11-09T22:56:49.836701140Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:56:49.836+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=776","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570609836]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AylNtosBOvK4VyD1lHI6","_score":null,"_source":{"@timestamp":"2023-11-09T22:56:52.841432821Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:56:52.841+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=777","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570612841]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BClNtosBOvK4VyD1lHI6","_score":null,"_source":{"@timestamp":"2023-11-09T22:56:55.842027510Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:56:55.841+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=778","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570615842]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xClNtosBOvK4VyD1rI1Z","_score":null,"_source":{"@timestamp":"2023-11-09T22:56:58.843832589Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:56:58.843+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=779","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570618843]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xSlNtosBOvK4VyD1rI1Z","_score":null,"_source":{"@timestamp":"2023-11-09T22:57:01.845918189Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:57:01.845+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=780","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570621845]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jSlNtosBOvK4VyD1xKxm","_score":null,"_source":{"@timestamp":"2023-11-09T22:57:04.847878628Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:57:04.847+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=781","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570624847]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jilNtosBOvK4VyD1xKxm","_score":null,"_source":{"@timestamp":"2023-11-09T22:57:07.849773207Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:57:07.849+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=782","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570627849]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jylNtosBOvK4VyD1xKxm","_score":null,"_source":{"@timestamp":"2023-11-09T22:57:10.851740561Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:57:10.851+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=783","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570630851]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gilNtosBOvK4VyD15NSM","_score":null,"_source":{"@timestamp":"2023-11-09T22:57:13.853548206Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:57:13.853+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=784","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570633853]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gylNtosBOvK4VyD15NSM","_score":null,"_source":{"@timestamp":"2023-11-09T22:57:16.855227536Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:57:16.854+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=785","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570636855]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YypNtosBOvK4VyD1_ACi","_score":null,"_source":{"@timestamp":"2023-11-09T22:57:19.856707457Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:57:19.856+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=786","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570639856]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZCpNtosBOvK4VyD1_ACi","_score":null,"_source":{"@timestamp":"2023-11-09T22:57:22.858216714Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:57:22.857+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=787","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570642858]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3ipOtosBOvK4VyD1FEq7","_score":null,"_source":{"@timestamp":"2023-11-09T22:57:25.863959Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:57:25.863+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=788","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570645863]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3ypOtosBOvK4VyD1FEq7","_score":null,"_source":{"@timestamp":"2023-11-09T22:57:28.865929952Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:57:28.865+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=789","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570648865]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XSpOtosBOvK4VyD1LJvQ","_score":null,"_source":{"@timestamp":"2023-11-09T22:57:31.867079196Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:57:31.866+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=790","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570651867]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XipOtosBOvK4VyD1LJvQ","_score":null,"_source":{"@timestamp":"2023-11-09T22:57:34.868817612Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:57:34.868+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=791","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570654868]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nCpOtosBOvK4VyD1ROHo","_score":null,"_source":{"@timestamp":"2023-11-09T22:57:37.870445678Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:57:37.869+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=792","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570657870]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nSpOtosBOvK4VyD1ROHo","_score":null,"_source":{"@timestamp":"2023-11-09T22:57:40.871989320Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:57:40.871+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=793","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570660871]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nStOtosBOvK4VyD1XRAD","_score":null,"_source":{"@timestamp":"2023-11-09T22:57:43.873329997Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:57:43.872+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=794","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570663873]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nitOtosBOvK4VyD1XRAD","_score":null,"_source":{"@timestamp":"2023-11-09T22:57:46.874535737Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:57:46.874+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=795","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570666874]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hStOtosBOvK4VyD1cScf","_score":null,"_source":{"@timestamp":"2023-11-09T22:57:49.876235108Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:57:49.875+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=796","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570669876]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hitOtosBOvK4VyD1cScf","_score":null,"_source":{"@timestamp":"2023-11-09T22:57:52.878005397Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:57:52.877+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=797","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570672878]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7CtOtosBOvK4VyD1iUU0","_score":null,"_source":{"@timestamp":"2023-11-09T22:57:55.879428876Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:57:55.879+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=798","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570675879]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7StOtosBOvK4VyD1iUU0","_score":null,"_source":{"@timestamp":"2023-11-09T22:57:58.885600450Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:57:58.885+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=799","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570678885]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OCtOtosBOvK4VyD1oWlM","_score":null,"_source":{"@timestamp":"2023-11-09T22:58:01.887290959Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:58:01.886+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=800","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570681887]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PStOtosBOvK4VyD1oWlM","_score":null,"_source":{"@timestamp":"2023-11-09T22:58:03.865821790Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:58:03.865+00:00 Cleaning up query iterator tid=1 + count=805","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570683865]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OStOtosBOvK4VyD1oWlM","_score":null,"_source":{"@timestamp":"2023-11-09T22:58:03.865656068Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:58:03.865+00:00 Cleaning up expired records tid=1 + count=801","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570683865]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OitOtosBOvK4VyD1oWlM","_score":null,"_source":{"@timestamp":"2023-11-09T22:58:03.865785459Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:58:03.865+00:00 Cleaned up expired records tid=1 + size=0 count=802","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570683865]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OytOtosBOvK4VyD1oWlM","_score":null,"_source":{"@timestamp":"2023-11-09T22:58:03.865797481Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:58:03.865+00:00 Cleaning up query iterator tid=1 + count=803","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570683865]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PCtOtosBOvK4VyD1oWlM","_score":null,"_source":{"@timestamp":"2023-11-09T22:58:03.865803585Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:58:03.865+00:00 Cleaning up orphaned records tid=1 + count=804","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570683865]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PitOtosBOvK4VyD1oWlM","_score":null,"_source":{"@timestamp":"2023-11-09T22:58:04.887667068Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:58:04.887+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=806","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570684887]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OCtOtosBOvK4VyD1uaRs","_score":null,"_source":{"@timestamp":"2023-11-09T22:58:07.888270987Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:58:07.887+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=807","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570687888]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OStOtosBOvK4VyD1uaRs","_score":null,"_source":{"@timestamp":"2023-11-09T22:58:10.890043855Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:58:10.889+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=808","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570690890]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sytOtosBOvK4VyD10diG","_score":null,"_source":{"@timestamp":"2023-11-09T22:58:13.891453324Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:58:13.891+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=809","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570693891]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tCtOtosBOvK4VyD10diG","_score":null,"_source":{"@timestamp":"2023-11-09T22:58:16.892755161Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:58:16.892+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=810","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570696892]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EitOtosBOvK4VyD16fec","_score":null,"_source":{"@timestamp":"2023-11-09T22:58:19.893593942Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:58:19.893+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=811","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570699893]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EytOtosBOvK4VyD16fec","_score":null,"_source":{"@timestamp":"2023-11-09T22:58:22.895101809Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:58:22.894+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=812","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570702895]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FCtOtosBOvK4VyD16fec","_score":null,"_source":{"@timestamp":"2023-11-09T22:58:25.895655434Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:58:25.895+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=813","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570705895]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rixPtosBOvK4VyD1CVLC","_score":null,"_source":{"@timestamp":"2023-11-09T22:58:28.897621619Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:58:28.897+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=814","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570708897]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ryxPtosBOvK4VyD1CVLC","_score":null,"_source":{"@timestamp":"2023-11-09T22:58:31.899279802Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:58:31.898+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=815","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570711899]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0SxPtosBOvK4VyD1IanY","_score":null,"_source":{"@timestamp":"2023-11-09T22:58:34.900732222Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:58:34.900+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=816","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570714900]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0ixPtosBOvK4VyD1IanY","_score":null,"_source":{"@timestamp":"2023-11-09T22:58:37.902418319Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:58:37.902+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=817","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570717902]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hS1PtosBOvK4VyD1OQDy","_score":null,"_source":{"@timestamp":"2023-11-09T22:58:40.908968707Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:58:40.908+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=818","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570720908]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hi1PtosBOvK4VyD1OQDy","_score":null,"_source":{"@timestamp":"2023-11-09T22:58:43.910531073Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:58:43.910+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=819","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570723910]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vC1PtosBOvK4VyD1Ul4N","_score":null,"_source":{"@timestamp":"2023-11-09T22:58:46.911302041Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:58:46.910+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=820","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570726911]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vS1PtosBOvK4VyD1Ul4N","_score":null,"_source":{"@timestamp":"2023-11-09T22:58:49.912962367Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:58:49.912+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=821","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570729912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6y1PtosBOvK4VyD1apom","_score":null,"_source":{"@timestamp":"2023-11-09T22:58:52.914619642Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:58:52.914+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=822","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570732914]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7C1PtosBOvK4VyD1apom","_score":null,"_source":{"@timestamp":"2023-11-09T22:58:55.915075826Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:58:55.914+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=823","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570735915]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wy1PtosBOvK4VyD1grc6","_score":null,"_source":{"@timestamp":"2023-11-09T22:58:58.917244842Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:58:58.916+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=824","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570738917]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xC1PtosBOvK4VyD1grc6","_score":null,"_source":{"@timestamp":"2023-11-09T22:59:01.918994251Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:59:01.918+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=825","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570741918]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ry1PtosBOvK4VyD1ltFO","_score":null,"_source":{"@timestamp":"2023-11-09T22:59:04.920504234Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:59:04.920+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=826","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570744920]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sC1PtosBOvK4VyD1ltFO","_score":null,"_source":{"@timestamp":"2023-11-09T22:59:07.927291482Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:59:07.926+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=827","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570747927]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3i1PtosBOvK4VyD1ru5p","_score":null,"_source":{"@timestamp":"2023-11-09T22:59:10.929419894Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:59:10.929+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=828","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570750929]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3y1PtosBOvK4VyD1ru5p","_score":null,"_source":{"@timestamp":"2023-11-09T22:59:13.930796172Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:59:13.930+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=829","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570753930]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Gi5PtosBOvK4VyD1xgyD","_score":null,"_source":{"@timestamp":"2023-11-09T22:59:16.932404253Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:59:16.931+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=830","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570756932]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Gy5PtosBOvK4VyD1xgyD","_score":null,"_source":{"@timestamp":"2023-11-09T22:59:19.933960574Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:59:19.933+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=831","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570759933]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ui5PtosBOvK4VyD13kyd","_score":null,"_source":{"@timestamp":"2023-11-09T22:59:22.935272569Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:59:22.934+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=832","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570762935]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Uy5PtosBOvK4VyD13kyd","_score":null,"_source":{"@timestamp":"2023-11-09T22:59:25.937129685Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:59:25.936+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=833","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570765937]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0C5PtosBOvK4VyD19pa2","_score":null,"_source":{"@timestamp":"2023-11-09T22:59:28.938011994Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:59:28.937+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=834","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570768938]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0S5PtosBOvK4VyD19pa2","_score":null,"_source":{"@timestamp":"2023-11-09T22:59:31.939217509Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:59:31.938+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=835","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570771939]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4S5QtosBOvK4VyD1DuvO","_score":null,"_source":{"@timestamp":"2023-11-09T22:59:34.944755377Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:59:34.944+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=836","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570774944]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4i5QtosBOvK4VyD1DuvO","_score":null,"_source":{"@timestamp":"2023-11-09T22:59:37.946236035Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:59:37.945+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=837","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570777946]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4y5QtosBOvK4VyD1DuvO","_score":null,"_source":{"@timestamp":"2023-11-09T22:59:40.947348737Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:59:40.947+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=838","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570780947]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aS9QtosBOvK4VyD1LjTy","_score":null,"_source":{"@timestamp":"2023-11-09T22:59:43.949414310Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:59:43.949+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=839","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570783949]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ai9QtosBOvK4VyD1LjTy","_score":null,"_source":{"@timestamp":"2023-11-09T22:59:46.950642887Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:59:46.950+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=840","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570786950]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mS9QtosBOvK4VyD1R1EI","_score":null,"_source":{"@timestamp":"2023-11-09T22:59:49.951596921Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:59:49.951+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=841","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570789951]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mi9QtosBOvK4VyD1R1EI","_score":null,"_source":{"@timestamp":"2023-11-09T22:59:52.952986845Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:59:52.952+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=842","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570792952]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uS9QtosBOvK4VyD1X24o","_score":null,"_source":{"@timestamp":"2023-11-09T22:59:55.954688385Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:59:55.954+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=843","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570795954]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ui9QtosBOvK4VyD1X24o","_score":null,"_source":{"@timestamp":"2023-11-09T22:59:58.956505344Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 22:59:58.956+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=844","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570798956]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yC9QtosBOvK4VyD1d4hO","_score":null,"_source":{"@timestamp":"2023-11-09T23:00:01.958781665Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:00:01.957+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=845","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570801958]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yS9QtosBOvK4VyD1d4hO","_score":null,"_source":{"@timestamp":"2023-11-09T23:00:04.960021338Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:00:04.959+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=846","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570804960]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JS9QtosBOvK4VyD1j7pk","_score":null,"_source":{"@timestamp":"2023-11-09T23:00:07.966750146Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:00:07.961+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=847","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570807966]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ji9QtosBOvK4VyD1j7pk","_score":null,"_source":{"@timestamp":"2023-11-09T23:00:10.963764748Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:00:10.963+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=848","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570810963]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LS9QtosBOvK4VyD1p-1v","_score":null,"_source":{"@timestamp":"2023-11-09T23:00:13.965819840Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:00:13.965+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=849","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570813965]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Li9QtosBOvK4VyD1p-1v","_score":null,"_source":{"@timestamp":"2023-11-09T23:00:16.967432022Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:00:16.967+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=850","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570816967]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cDBQtosBOvK4VyD1uxCK","_score":null,"_source":{"@timestamp":"2023-11-09T23:00:19.969430087Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:00:19.969+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=851","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570819969]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cTBQtosBOvK4VyD1uxCK","_score":null,"_source":{"@timestamp":"2023-11-09T23:00:22.970999687Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:00:22.970+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=852","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570822970]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ujBQtosBOvK4VyD10zCj","_score":null,"_source":{"@timestamp":"2023-11-09T23:00:25.972604503Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:00:25.972+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=853","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570825972]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uzBQtosBOvK4VyD10zCj","_score":null,"_source":{"@timestamp":"2023-11-09T23:00:28.978772859Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:00:28.978+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=854","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570828978]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SjBQtosBOvK4VyD163jB","_score":null,"_source":{"@timestamp":"2023-11-09T23:00:31.980219890Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:00:31.979+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=855","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570831980]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SzBQtosBOvK4VyD163jB","_score":null,"_source":{"@timestamp":"2023-11-09T23:00:34.980648874Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:00:34.980+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=856","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570834980]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gDBRtosBOvK4VyD1A8rS","_score":null,"_source":{"@timestamp":"2023-11-09T23:00:37.982242683Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:00:37.981+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=857","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570837982]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gTBRtosBOvK4VyD1A8rS","_score":null,"_source":{"@timestamp":"2023-11-09T23:00:40.983049298Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:00:40.982+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=858","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570840983]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XjFRtosBOvK4VyD1GyHr","_score":null,"_source":{"@timestamp":"2023-11-09T23:00:43.983665878Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:00:43.983+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=859","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570843983]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XzFRtosBOvK4VyD1GyHr","_score":null,"_source":{"@timestamp":"2023-11-09T23:00:46.984938916Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:00:46.984+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=860","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570846984]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1jFRtosBOvK4VyD1NGsE","_score":null,"_source":{"@timestamp":"2023-11-09T23:00:49.985899302Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:00:49.985+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=861","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570849985]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1zFRtosBOvK4VyD1NGsE","_score":null,"_source":{"@timestamp":"2023-11-09T23:00:52.987275100Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:00:52.986+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=862","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570852987]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vTFRtosBOvK4VyD1SLcg","_score":null,"_source":{"@timestamp":"2023-11-09T23:00:55.989329156Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:00:55.988+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=863","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570855989]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vjFRtosBOvK4VyD1SLcg","_score":null,"_source":{"@timestamp":"2023-11-09T23:00:58.991250661Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:00:58.990+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=864","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570858991]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zzFRtosBOvK4VyD1YN80","_score":null,"_source":{"@timestamp":"2023-11-09T23:01:01.993386131Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:01:01.992+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=865","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570861993]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0DFRtosBOvK4VyD1YN80","_score":null,"_source":{"@timestamp":"2023-11-09T23:01:04.995408108Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:01:04.995+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=866","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570864995]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6DFRtosBOvK4VyD1ePxN","_score":null,"_source":{"@timestamp":"2023-11-09T23:01:07.997009646Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:01:07.996+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=867","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570867997]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6TFRtosBOvK4VyD1ePxN","_score":null,"_source":{"@timestamp":"2023-11-09T23:01:11.003227523Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:01:11.002+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=868","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570871003]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_zJRtosBOvK4VyD1kBli","_score":null,"_source":{"@timestamp":"2023-11-09T23:01:14.004224225Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:01:14.003+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=869","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570874004]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ADJRtosBOvK4VyD1kBpi","_score":null,"_source":{"@timestamp":"2023-11-09T23:01:17.005180168Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:01:17.004+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=870","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570877005]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oDJRtosBOvK4VyD1qEl7","_score":null,"_source":{"@timestamp":"2023-11-09T23:01:20.005789981Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:01:20.005+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=871","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570880005]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oTJRtosBOvK4VyD1qEl7","_score":null,"_source":{"@timestamp":"2023-11-09T23:01:23.006184047Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:01:23.005+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=872","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570883006]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4TJRtosBOvK4VyD1wJKT","_score":null,"_source":{"@timestamp":"2023-11-09T23:01:26.006767874Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:01:26.006+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=873","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570886006]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4jJRtosBOvK4VyD1wJKT","_score":null,"_source":{"@timestamp":"2023-11-09T23:01:29.007794094Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:01:29.007+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=874","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570889007]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4zJRtosBOvK4VyD1wJKT","_score":null,"_source":{"@timestamp":"2023-11-09T23:01:32.008324097Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:01:32.007+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=875","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570892008]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2DNRtosBOvK4VyD14AK0","_score":null,"_source":{"@timestamp":"2023-11-09T23:01:35.008673539Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:01:35.008+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=876","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570895008]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2TNRtosBOvK4VyD14AK0","_score":null,"_source":{"@timestamp":"2023-11-09T23:01:38.009302040Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:01:38.008+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=877","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570898009]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ljNRtosBOvK4VyD1-DvM","_score":null,"_source":{"@timestamp":"2023-11-09T23:01:41.010060100Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:01:41.009+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=878","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570901010]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lzNRtosBOvK4VyD1-DvM","_score":null,"_source":{"@timestamp":"2023-11-09T23:01:44.010757966Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:01:44.010+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=879","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570904010]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hTNStosBOvK4VyD1EGHl","_score":null,"_source":{"@timestamp":"2023-11-09T23:01:47.011306408Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:01:47.010+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=880","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570907011]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hjNStosBOvK4VyD1EGHl","_score":null,"_source":{"@timestamp":"2023-11-09T23:01:50.012283501Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:01:50.011+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=881","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570910012]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NjNStosBOvK4VyD1KH76","_score":null,"_source":{"@timestamp":"2023-11-09T23:01:53.013419233Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:01:53.013+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=882","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570913013]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NzNStosBOvK4VyD1KH76","_score":null,"_source":{"@timestamp":"2023-11-09T23:01:56.014136609Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:01:56.013+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=883","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570916014]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HDNStosBOvK4VyD1QZoP","_score":null,"_source":{"@timestamp":"2023-11-09T23:01:59.014941939Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:01:59.014+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=884","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570919014]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HTNStosBOvK4VyD1QZoP","_score":null,"_source":{"@timestamp":"2023-11-09T23:02:02.016595566Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:02:02.015+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=885","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570922016]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PDNStosBOvK4VyD1Wc0s","_score":null,"_source":{"@timestamp":"2023-11-09T23:02:05.016728165Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:02:05.016+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=886","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570925016]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PTNStosBOvK4VyD1Wc0s","_score":null,"_source":{"@timestamp":"2023-11-09T23:02:08.017388052Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:02:08.017+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=887","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570928017]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KDNStosBOvK4VyD1bf1H","_score":null,"_source":{"@timestamp":"2023-11-09T23:02:11.018241677Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:02:11.017+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=888","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570931018]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KTNStosBOvK4VyD1bf1H","_score":null,"_source":{"@timestamp":"2023-11-09T23:02:14.019220682Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:02:14.018+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=889","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570934019]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ojRStosBOvK4VyD1hSla","_score":null,"_source":{"@timestamp":"2023-11-09T23:02:17.020566662Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:02:17.020+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=890","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570937020]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ozRStosBOvK4VyD1hSla","_score":null,"_source":{"@timestamp":"2023-11-09T23:02:20.021370120Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:02:20.020+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=891","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570940021]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tDRStosBOvK4VyD1nUZw","_score":null,"_source":{"@timestamp":"2023-11-09T23:02:23.022399751Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:02:23.021+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=892","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570943022]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tTRStosBOvK4VyD1nUZw","_score":null,"_source":{"@timestamp":"2023-11-09T23:02:26.023076326Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:02:26.022+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=893","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570946023]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fTRStosBOvK4VyD1tZeH","_score":null,"_source":{"@timestamp":"2023-11-09T23:02:29.024063398Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:02:29.023+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=894","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570949024]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fjRStosBOvK4VyD1tZeH","_score":null,"_source":{"@timestamp":"2023-11-09T23:02:32.024676108Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:02:32.024+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=895","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570952024]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gjRStosBOvK4VyD1zemu","_score":null,"_source":{"@timestamp":"2023-11-09T23:02:35.025388654Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:02:35.025+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=896","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570955025]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gzRStosBOvK4VyD1zemu","_score":null,"_source":{"@timestamp":"2023-11-09T23:02:38.026228510Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:02:38.025+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=897","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570958026]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AzVStosBOvK4VyD15T-2","_score":null,"_source":{"@timestamp":"2023-11-09T23:02:41.026533796Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:02:41.026+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=898","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570961026]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BDVStosBOvK4VyD15T-2","_score":null,"_source":{"@timestamp":"2023-11-09T23:02:44.027232679Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:02:44.026+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=899","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570964027]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BTVStosBOvK4VyD15T-2","_score":null,"_source":{"@timestamp":"2023-11-09T23:02:47.032190514Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:02:47.031+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=900","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570967032]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mzVTtosBOvK4VyD1BbDY","_score":null,"_source":{"@timestamp":"2023-11-09T23:02:50.033262265Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:02:50.032+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=901","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570970033]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nDVTtosBOvK4VyD1BbDY","_score":null,"_source":{"@timestamp":"2023-11-09T23:02:53.034506442Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:02:53.034+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=902","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570973034]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SDVTtosBOvK4VyD1Hejx","_score":null,"_source":{"@timestamp":"2023-11-09T23:02:56.035533035Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:02:56.035+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=903","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570976035]},{"_index":"continuous-tests-pods-2023.11.09","_id":"STVTtosBOvK4VyD1Hejx","_score":null,"_source":{"@timestamp":"2023-11-09T23:02:59.036194738Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:02:59.035+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=904","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570979036]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rTZTtosBOvK4VyD1NgYL","_score":null,"_source":{"@timestamp":"2023-11-09T23:03:02.037013760Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:03:02.036+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=905","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570982037]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rjZTtosBOvK4VyD1NgYL","_score":null,"_source":{"@timestamp":"2023-11-09T23:03:03.867748434Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:03:03.867+00:00 Cleaning up expired records tid=1 + count=906","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570983867]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rzZTtosBOvK4VyD1NgYL","_score":null,"_source":{"@timestamp":"2023-11-09T23:03:03.867805747Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:03:03.867+00:00 Cleaned up expired records tid=1 + size=0 count=907","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570983867]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sDZTtosBOvK4VyD1NgYL","_score":null,"_source":{"@timestamp":"2023-11-09T23:03:03.867832144Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:03:03.867+00:00 Cleaning up query iterator tid=1 + count=908","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570983867]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sTZTtosBOvK4VyD1NgYL","_score":null,"_source":{"@timestamp":"2023-11-09T23:03:03.867836171Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:03:03.867+00:00 Cleaning up orphaned records tid=1 + count=909","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570983867]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sjZTtosBOvK4VyD1NgYL","_score":null,"_source":{"@timestamp":"2023-11-09T23:03:03.867985146Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:03:03.867+00:00 Cleaning up query iterator tid=1 + count=910","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570983867]},{"_index":"continuous-tests-pods-2023.11.09","_id":"szZTtosBOvK4VyD1NgYL","_score":null,"_source":{"@timestamp":"2023-11-09T23:03:05.037515079Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:03:05.037+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=911","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570985037]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rzZTtosBOvK4VyD1TCAp","_score":null,"_source":{"@timestamp":"2023-11-09T23:03:08.042963900Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:03:08.042+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=912","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570988042]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sDZTtosBOvK4VyD1TCAp","_score":null,"_source":{"@timestamp":"2023-11-09T23:03:11.043351675Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:03:11.043+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=913","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570991043]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UDZTtosBOvK4VyD1ZD9A","_score":null,"_source":{"@timestamp":"2023-11-09T23:03:14.044000157Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:03:14.043+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=914","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570994044]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UTZTtosBOvK4VyD1ZD9A","_score":null,"_source":{"@timestamp":"2023-11-09T23:03:17.044605069Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:03:17.044+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=915","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699570997044]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QTZTtosBOvK4VyD1fGla","_score":null,"_source":{"@timestamp":"2023-11-09T23:03:20.045554761Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:03:20.045+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=916","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571000045]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QjZTtosBOvK4VyD1fGla","_score":null,"_source":{"@timestamp":"2023-11-09T23:03:23.046705039Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:03:23.046+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=917","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571003046]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AjZTtosBOvK4VyD1lLFu","_score":null,"_source":{"@timestamp":"2023-11-09T23:03:26.047657486Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:03:26.047+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=918","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571006047]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AzZTtosBOvK4VyD1lLFu","_score":null,"_source":{"@timestamp":"2023-11-09T23:03:29.048182996Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:03:29.047+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=919","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571009048]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HDZTtosBOvK4VyD1rPaH","_score":null,"_source":{"@timestamp":"2023-11-09T23:03:32.048668265Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:03:32.048+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=920","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571012048]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HTZTtosBOvK4VyD1rPaH","_score":null,"_source":{"@timestamp":"2023-11-09T23:03:35.053642821Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:03:35.053+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=921","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571015053]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0DdTtosBOvK4VyD1xD6j","_score":null,"_source":{"@timestamp":"2023-11-09T23:03:38.054453514Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:03:38.054+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=922","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571018054]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0TdTtosBOvK4VyD1xD6j","_score":null,"_source":{"@timestamp":"2023-11-09T23:03:41.054578607Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:03:41.054+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=923","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571021054]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nDdTtosBOvK4VyD12HS7","_score":null,"_source":{"@timestamp":"2023-11-09T23:03:44.055858461Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:03:44.055+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=924","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571024055]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nTdTtosBOvK4VyD12HS7","_score":null,"_source":{"@timestamp":"2023-11-09T23:03:47.056678945Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:03:47.056+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=925","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571027056]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rjdTtosBOvK4VyD18JHY","_score":null,"_source":{"@timestamp":"2023-11-09T23:03:50.057416943Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:03:50.057+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=926","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571030057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rzdTtosBOvK4VyD18JHY","_score":null,"_source":{"@timestamp":"2023-11-09T23:03:53.058425216Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:03:53.058+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=927","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571033058]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oDdUtosBOvK4VyD1CK7r","_score":null,"_source":{"@timestamp":"2023-11-09T23:03:56.058779348Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:03:56.058+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=928","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571036058]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oTdUtosBOvK4VyD1CK7r","_score":null,"_source":{"@timestamp":"2023-11-09T23:03:59.059933006Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:03:59.059+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=929","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571039059]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KjdUtosBOvK4VyD1Ic0H","_score":null,"_source":{"@timestamp":"2023-11-09T23:04:02.060371848Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:04:02.059+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=930","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571042060]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KzdUtosBOvK4VyD1Ic0H","_score":null,"_source":{"@timestamp":"2023-11-09T23:04:05.061230373Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:04:05.060+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=931","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571045061]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OzhUtosBOvK4VyD1OQMh","_score":null,"_source":{"@timestamp":"2023-11-09T23:04:08.066856494Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:04:08.066+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=932","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571048066]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PDhUtosBOvK4VyD1OQMh","_score":null,"_source":{"@timestamp":"2023-11-09T23:04:11.067773922Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:04:11.067+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=933","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571051067]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pDhUtosBOvK4VyD1UTw5","_score":null,"_source":{"@timestamp":"2023-11-09T23:04:14.068551472Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:04:14.068+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=934","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571054068]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pThUtosBOvK4VyD1UTw5","_score":null,"_source":{"@timestamp":"2023-11-09T23:04:17.069756055Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:04:17.069+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=935","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571057069]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0zhUtosBOvK4VyD1ZVxN","_score":null,"_source":{"@timestamp":"2023-11-09T23:04:20.070322746Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:04:20.070+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=936","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571060070]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1DhUtosBOvK4VyD1ZVxN","_score":null,"_source":{"@timestamp":"2023-11-09T23:04:23.070947564Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:04:23.070+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=937","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571063070]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3zhUtosBOvK4VyD1fXlm","_score":null,"_source":{"@timestamp":"2023-11-09T23:04:26.071987674Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:04:26.071+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=938","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571066071]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4DhUtosBOvK4VyD1fXlm","_score":null,"_source":{"@timestamp":"2023-11-09T23:04:29.073004463Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:04:29.072+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=939","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571069073]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HDhUtosBOvK4VyD1ls9l","_score":null,"_source":{"@timestamp":"2023-11-09T23:04:32.073744623Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:04:32.073+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=940","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571072073]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HThUtosBOvK4VyD1ls9l","_score":null,"_source":{"@timestamp":"2023-11-09T23:04:35.074565765Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:04:35.074+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=941","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571075074]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dTlUtosBOvK4VyD1riR9","_score":null,"_source":{"@timestamp":"2023-11-09T23:04:38.075241124Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:04:38.074+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=942","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571078075]},{"_index":"continuous-tests-pods-2023.11.09","_id":"djlUtosBOvK4VyD1riR9","_score":null,"_source":{"@timestamp":"2023-11-09T23:04:41.076389031Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:04:41.075+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=943","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571081076]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2DlUtosBOvK4VyD1xnaZ","_score":null,"_source":{"@timestamp":"2023-11-09T23:04:44.077410389Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:04:44.077+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=944","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571084077]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2TlUtosBOvK4VyD1xnaZ","_score":null,"_source":{"@timestamp":"2023-11-09T23:04:47.077954230Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:04:47.077+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=945","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571087077]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1DlUtosBOvK4VyD12rqx","_score":null,"_source":{"@timestamp":"2023-11-09T23:04:50.078214163Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:04:50.077+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=946","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571090078]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1TlUtosBOvK4VyD12rqx","_score":null,"_source":{"@timestamp":"2023-11-09T23:04:53.079279454Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:04:53.078+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=947","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571093079]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iDpUtosBOvK4VyD18gjJ","_score":null,"_source":{"@timestamp":"2023-11-09T23:04:56.080356315Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:04:56.080+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=948","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571096080]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iTpUtosBOvK4VyD18gjJ","_score":null,"_source":{"@timestamp":"2023-11-09T23:04:59.085198490Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:04:59.084+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=949","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571099085]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HDpVtosBOvK4VyD1Cini","_score":null,"_source":{"@timestamp":"2023-11-09T23:05:02.085749789Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:05:02.085+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=950","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571102085]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HTpVtosBOvK4VyD1Cini","_score":null,"_source":{"@timestamp":"2023-11-09T23:05:05.086960114Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:05:05.086+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=951","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571105086]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lzpVtosBOvK4VyD1I0YB","_score":null,"_source":{"@timestamp":"2023-11-09T23:05:08.088091501Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:05:08.087+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=952","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571108088]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mDpVtosBOvK4VyD1I0YB","_score":null,"_source":{"@timestamp":"2023-11-09T23:05:11.088925289Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:05:11.088+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=953","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571111088]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lTpVtosBOvK4VyD1O2Me","_score":null,"_source":{"@timestamp":"2023-11-09T23:05:14.089962870Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:05:14.089+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=954","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571114089]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ljpVtosBOvK4VyD1O2Me","_score":null,"_source":{"@timestamp":"2023-11-09T23:05:17.091278057Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:05:17.090+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=955","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571117091]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FzpVtosBOvK4VyD1U4g0","_score":null,"_source":{"@timestamp":"2023-11-09T23:05:20.091932633Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:05:20.091+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=956","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571120091]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GDpVtosBOvK4VyD1U4g0","_score":null,"_source":{"@timestamp":"2023-11-09T23:05:23.092578289Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:05:23.092+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=957","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571123092]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GzpVtosBOvK4VyD1Z8tM","_score":null,"_source":{"@timestamp":"2023-11-09T23:05:26.093465251Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:05:26.093+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=958","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571126093]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HDpVtosBOvK4VyD1Z8tM","_score":null,"_source":{"@timestamp":"2023-11-09T23:05:29.094292846Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:05:29.093+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=959","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571129094]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rTtVtosBOvK4VyD1fxJl","_score":null,"_source":{"@timestamp":"2023-11-09T23:05:32.095161271Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:05:32.094+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=960","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571132095]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rjtVtosBOvK4VyD1fxJl","_score":null,"_source":{"@timestamp":"2023-11-09T23:05:35.096088939Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:05:35.095+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=961","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571135096]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fTtVtosBOvK4VyD1l15_","_score":null,"_source":{"@timestamp":"2023-11-09T23:05:38.096703460Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:05:38.096+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=962","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571138096]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fjtVtosBOvK4VyD1l15_","_score":null,"_source":{"@timestamp":"2023-11-09T23:05:41.097133246Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:05:41.096+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=963","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571141097]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PTtVtosBOvK4VyD1r5mX","_score":null,"_source":{"@timestamp":"2023-11-09T23:05:44.097420418Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:05:44.097+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=964","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571144097]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PjtVtosBOvK4VyD1r5mX","_score":null,"_source":{"@timestamp":"2023-11-09T23:05:47.098410206Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:05:47.098+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=965","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571147098]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kztVtosBOvK4VyD1x7au","_score":null,"_source":{"@timestamp":"2023-11-09T23:05:50.099563919Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:05:50.099+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=966","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571150099]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lDtVtosBOvK4VyD1x7au","_score":null,"_source":{"@timestamp":"2023-11-09T23:05:53.100203458Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:05:53.099+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=967","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571153100]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qDtVtosBOvK4VyD139PG","_score":null,"_source":{"@timestamp":"2023-11-09T23:05:56.101389846Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:05:56.101+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=968","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571156101]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qTtVtosBOvK4VyD139PG","_score":null,"_source":{"@timestamp":"2023-11-09T23:05:59.102583539Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:05:59.102+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=969","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571159102]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ujtVtosBOvK4VyD18-7g","_score":null,"_source":{"@timestamp":"2023-11-09T23:06:02.103086088Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:06:02.102+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=970","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571162103]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uztVtosBOvK4VyD18-7g","_score":null,"_source":{"@timestamp":"2023-11-09T23:06:05.103638298Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:06:05.103+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=971","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571165103]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-zxWtosBOvK4VyD1Cyf6","_score":null,"_source":{"@timestamp":"2023-11-09T23:06:08.104608804Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:06:08.104+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=972","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571168104]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_DxWtosBOvK4VyD1Cyf6","_score":null,"_source":{"@timestamp":"2023-11-09T23:06:11.105789297Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:06:11.105+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=973","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571171105]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DDxWtosBOvK4VyD1JGET","_score":null,"_source":{"@timestamp":"2023-11-09T23:06:14.107062846Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:06:14.106+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=974","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571174107]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DTxWtosBOvK4VyD1JGET","_score":null,"_source":{"@timestamp":"2023-11-09T23:06:17.108304660Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:06:17.107+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=975","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571177108]},{"_index":"continuous-tests-pods-2023.11.09","_id":"azxWtosBOvK4VyD1PIIq","_score":null,"_source":{"@timestamp":"2023-11-09T23:06:20.108741058Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:06:20.108+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=976","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571180108]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bDxWtosBOvK4VyD1PIIq","_score":null,"_source":{"@timestamp":"2023-11-09T23:06:23.109655178Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:06:23.109+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=977","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571183109]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xzxWtosBOvK4VyD1VLBF","_score":null,"_source":{"@timestamp":"2023-11-09T23:06:26.110066120Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:06:26.109+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=978","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571186110]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yDxWtosBOvK4VyD1VLBF","_score":null,"_source":{"@timestamp":"2023-11-09T23:06:29.110713397Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:06:29.110+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=979","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571189110]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JD1WtosBOvK4VyD1bAtb","_score":null,"_source":{"@timestamp":"2023-11-09T23:06:32.111475351Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:06:32.111+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=980","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571192111]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JT1WtosBOvK4VyD1bAtb","_score":null,"_source":{"@timestamp":"2023-11-09T23:06:35.111923236Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:06:35.111+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=981","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571195111]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Jj1WtosBOvK4VyD1bAtb","_score":null,"_source":{"@timestamp":"2023-11-09T23:06:38.113180566Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:06:38.112+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=982","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571198113]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Sz1WtosBOvK4VyD1jIJ_","_score":null,"_source":{"@timestamp":"2023-11-09T23:06:41.118434362Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:06:41.118+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=983","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571201118]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TD1WtosBOvK4VyD1jIJ_","_score":null,"_source":{"@timestamp":"2023-11-09T23:06:44.119172553Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:06:44.118+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=984","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571204119]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Bz1WtosBOvK4VyD1pNya","_score":null,"_source":{"@timestamp":"2023-11-09T23:06:47.119769739Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:06:47.119+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=985","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571207119]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CD1WtosBOvK4VyD1pNya","_score":null,"_source":{"@timestamp":"2023-11-09T23:06:50.120976462Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:06:50.120+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=986","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571210120]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VD5WtosBOvK4VyD1vCOv","_score":null,"_source":{"@timestamp":"2023-11-09T23:06:53.122241566Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:06:53.121+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=987","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571213122]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VT5WtosBOvK4VyD1vCOv","_score":null,"_source":{"@timestamp":"2023-11-09T23:06:56.123050138Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:06:56.122+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=988","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571216123]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Xz5WtosBOvK4VyD11ELE","_score":null,"_source":{"@timestamp":"2023-11-09T23:06:59.123574938Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:06:59.123+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=989","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571219123]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YD5WtosBOvK4VyD11ELE","_score":null,"_source":{"@timestamp":"2023-11-09T23:07:02.124429368Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:07:02.124+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=990","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571222124]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TD5WtosBOvK4VyD17Fze","_score":null,"_source":{"@timestamp":"2023-11-09T23:07:05.125117441Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:07:05.124+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=991","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571225125]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TT5WtosBOvK4VyD17Fze","_score":null,"_source":{"@timestamp":"2023-11-09T23:07:08.130580841Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:07:08.130+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=992","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571228130]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Cj5XtosBOvK4VyD1BHj4","_score":null,"_source":{"@timestamp":"2023-11-09T23:07:11.131002534Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:07:11.130+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=993","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571231131]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Cz5XtosBOvK4VyD1BHj4","_score":null,"_source":{"@timestamp":"2023-11-09T23:07:14.131959124Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:07:14.131+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=994","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571234131]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0z5XtosBOvK4VyD1GY0N","_score":null,"_source":{"@timestamp":"2023-11-09T23:07:17.133227408Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:07:17.132+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=995","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571237133]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1D5XtosBOvK4VyD1GY0N","_score":null,"_source":{"@timestamp":"2023-11-09T23:07:20.133665522Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:07:20.133+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=996","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571240133]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1T5XtosBOvK4VyD1McMk","_score":null,"_source":{"@timestamp":"2023-11-09T23:07:23.134150462Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:07:23.133+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=997","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571243134]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1j5XtosBOvK4VyD1McMk","_score":null,"_source":{"@timestamp":"2023-11-09T23:07:26.135376659Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:07:26.135+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=998","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571246135]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FD9XtosBOvK4VyD1SRQ5","_score":null,"_source":{"@timestamp":"2023-11-09T23:07:29.140992825Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:07:29.140+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=999","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571249140]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FT9XtosBOvK4VyD1SRQ5","_score":null,"_source":{"@timestamp":"2023-11-09T23:07:32.142165724Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:07:32.141+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1000","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571252142]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FT9XtosBOvK4VyD1YWNQ","_score":null,"_source":{"@timestamp":"2023-11-09T23:07:35.143386966Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:07:35.143+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1001","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571255143]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Fj9XtosBOvK4VyD1YWNQ","_score":null,"_source":{"@timestamp":"2023-11-09T23:07:38.144526478Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:07:38.144+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1002","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571258144]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WT9XtosBOvK4VyD1eaBr","_score":null,"_source":{"@timestamp":"2023-11-09T23:07:41.145352700Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:07:41.144+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1003","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571261145]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Wj9XtosBOvK4VyD1eaBr","_score":null,"_source":{"@timestamp":"2023-11-09T23:07:44.146080804Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:07:44.145+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1004","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571264146]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2T9XtosBOvK4VyD1kbt9","_score":null,"_source":{"@timestamp":"2023-11-09T23:07:47.147272329Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:07:47.146+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1005","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571267147]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2j9XtosBOvK4VyD1kbt9","_score":null,"_source":{"@timestamp":"2023-11-09T23:07:50.147579910Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:07:50.147+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1006","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571270147]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2z9XtosBOvK4VyD1kbt9","_score":null,"_source":{"@timestamp":"2023-11-09T23:07:53.148836155Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:07:53.148+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1007","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571273148]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hD9XtosBOvK4VyD1seCl","_score":null,"_source":{"@timestamp":"2023-11-09T23:07:56.149635418Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:07:56.149+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1008","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571276149]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hT9XtosBOvK4VyD1seCl","_score":null,"_source":{"@timestamp":"2023-11-09T23:07:59.150788782Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:07:59.150+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1009","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571279150]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3z9XtosBOvK4VyD1yfq9","_score":null,"_source":{"@timestamp":"2023-11-09T23:08:02.151700391Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:08:02.151+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1010","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571282151]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4D9XtosBOvK4VyD1yfq9","_score":null,"_source":{"@timestamp":"2023-11-09T23:08:03.869130063Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:08:03.868+00:00 Cleaning up expired records tid=1 + count=1011","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571283869]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4T9XtosBOvK4VyD1yfq9","_score":null,"_source":{"@timestamp":"2023-11-09T23:08:03.869256324Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:08:03.869+00:00 Cleaned up expired records tid=1 + size=0 count=1012","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571283869]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4j9XtosBOvK4VyD1yfq9","_score":null,"_source":{"@timestamp":"2023-11-09T23:08:03.869293686Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:08:03.869+00:00 Cleaning up query iterator tid=1 + count=1013","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571283869]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4z9XtosBOvK4VyD1yfq9","_score":null,"_source":{"@timestamp":"2023-11-09T23:08:03.869300134Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:08:03.869+00:00 Cleaning up orphaned records tid=1 + count=1014","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571283869]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5D9XtosBOvK4VyD1yfq9","_score":null,"_source":{"@timestamp":"2023-11-09T23:08:03.869446149Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:08:03.869+00:00 Cleaning up query iterator tid=1 + count=1015","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571283869]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5T9XtosBOvK4VyD1yfq9","_score":null,"_source":{"@timestamp":"2023-11-09T23:08:05.152572276Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:08:05.152+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1016","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571285152]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3kBXtosBOvK4VyD14zjb","_score":null,"_source":{"@timestamp":"2023-11-09T23:08:08.153795667Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:08:08.153+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1017","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571288153]},{"_index":"continuous-tests-pods-2023.11.09","_id":"30BXtosBOvK4VyD14zjb","_score":null,"_source":{"@timestamp":"2023-11-09T23:08:11.154597902Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:08:11.154+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1018","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571291154]},{"_index":"continuous-tests-pods-2023.11.09","_id":"w0BXtosBOvK4VyD192j3","_score":null,"_source":{"@timestamp":"2023-11-09T23:08:14.155456971Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:08:14.155+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1019","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571294155]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xEBXtosBOvK4VyD192j3","_score":null,"_source":{"@timestamp":"2023-11-09T23:08:17.156398737Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:08:17.156+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1020","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571297156]},{"_index":"continuous-tests-pods-2023.11.09","_id":"REBYtosBOvK4VyD1EIcP","_score":null,"_source":{"@timestamp":"2023-11-09T23:08:20.156900556Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:08:20.156+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1021","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571300156]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RUBYtosBOvK4VyD1EIcP","_score":null,"_source":{"@timestamp":"2023-11-09T23:08:23.161861872Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:08:23.161+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1022","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571303161]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MkBYtosBOvK4VyD1KLIl","_score":null,"_source":{"@timestamp":"2023-11-09T23:08:26.162613834Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:08:26.162+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1023","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571306162]},{"_index":"continuous-tests-pods-2023.11.09","_id":"M0BYtosBOvK4VyD1KLIl","_score":null,"_source":{"@timestamp":"2023-11-09T23:08:29.163528123Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:08:29.163+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1024","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571309163]},{"_index":"continuous-tests-pods-2023.11.09","_id":"60FYtosBOvK4VyD1QAQ9","_score":null,"_source":{"@timestamp":"2023-11-09T23:08:32.164477482Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:08:32.164+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1025","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571312164]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7EFYtosBOvK4VyD1QAQ9","_score":null,"_source":{"@timestamp":"2023-11-09T23:08:35.165886527Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:08:35.165+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1026","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571315165]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-kFYtosBOvK4VyD1WFlV","_score":null,"_source":{"@timestamp":"2023-11-09T23:08:38.166914406Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:08:38.166+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1027","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571318166]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-0FYtosBOvK4VyD1WFlV","_score":null,"_source":{"@timestamp":"2023-11-09T23:08:41.167670147Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:08:41.167+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1028","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571321167]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fEFYtosBOvK4VyD1cK9r","_score":null,"_source":{"@timestamp":"2023-11-09T23:08:44.168920609Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:08:44.168+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1029","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571324168]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fUFYtosBOvK4VyD1cK9r","_score":null,"_source":{"@timestamp":"2023-11-09T23:08:47.170099675Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:08:47.169+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1030","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571327170]},{"_index":"continuous-tests-pods-2023.11.09","_id":"t0FYtosBOvK4VyD1hPiE","_score":null,"_source":{"@timestamp":"2023-11-09T23:08:50.176111920Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:08:50.175+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1031","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571330176]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uEFYtosBOvK4VyD1hPiE","_score":null,"_source":{"@timestamp":"2023-11-09T23:08:53.177296714Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:08:53.176+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1032","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571333177]},{"_index":"continuous-tests-pods-2023.11.09","_id":"P0JYtosBOvK4VyD1nCqc","_score":null,"_source":{"@timestamp":"2023-11-09T23:08:56.178203749Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:08:56.177+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1033","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571336178]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QEJYtosBOvK4VyD1nCqc","_score":null,"_source":{"@timestamp":"2023-11-09T23:08:59.178957463Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:08:59.178+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1034","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571339178]},{"_index":"continuous-tests-pods-2023.11.09","_id":"z0JYtosBOvK4VyD1tEW5","_score":null,"_source":{"@timestamp":"2023-11-09T23:09:02.179750128Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:09:02.179+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1035","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571342179]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0EJYtosBOvK4VyD1tEW5","_score":null,"_source":{"@timestamp":"2023-11-09T23:09:05.181221543Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:09:05.180+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1036","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571345181]},{"_index":"continuous-tests-pods-2023.11.09","_id":"B0JYtosBOvK4VyD1zGDR","_score":null,"_source":{"@timestamp":"2023-11-09T23:09:08.182262546Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:09:08.182+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1037","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571348182]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CEJYtosBOvK4VyD1zGDR","_score":null,"_source":{"@timestamp":"2023-11-09T23:09:11.182828523Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:09:11.182+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1038","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571351182]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5EJYtosBOvK4VyD15Hvm","_score":null,"_source":{"@timestamp":"2023-11-09T23:09:14.183852953Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:09:14.183+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1039","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571354183]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5UJYtosBOvK4VyD15Hvm","_score":null,"_source":{"@timestamp":"2023-11-09T23:09:17.189095708Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:09:17.188+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1040","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571357189]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2kJYtosBOvK4VyD1_aUA","_score":null,"_source":{"@timestamp":"2023-11-09T23:09:20.189375253Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:09:20.189+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1041","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571360189]},{"_index":"continuous-tests-pods-2023.11.09","_id":"20JYtosBOvK4VyD1_aUA","_score":null,"_source":{"@timestamp":"2023-11-09T23:09:23.190694597Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:09:23.190+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1042","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571363190]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wEJZtosBOvK4VyD1EeEX","_score":null,"_source":{"@timestamp":"2023-11-09T23:09:26.191111856Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:09:26.190+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1043","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571366191]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wUJZtosBOvK4VyD1EeEX","_score":null,"_source":{"@timestamp":"2023-11-09T23:09:29.191742096Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:09:29.191+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1044","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571369191]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vkNZtosBOvK4VyD1KTcu","_score":null,"_source":{"@timestamp":"2023-11-09T23:09:32.192109431Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:09:32.191+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1045","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571372192]},{"_index":"continuous-tests-pods-2023.11.09","_id":"v0NZtosBOvK4VyD1KTcu","_score":null,"_source":{"@timestamp":"2023-11-09T23:09:35.193319891Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:09:35.193+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1046","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571375193]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bUNZtosBOvK4VyD1QWxG","_score":null,"_source":{"@timestamp":"2023-11-09T23:09:38.198522659Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:09:38.198+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1047","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571378198]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bkNZtosBOvK4VyD1QWxG","_score":null,"_source":{"@timestamp":"2023-11-09T23:09:41.200076471Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:09:41.199+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1048","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571381200]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7ENZtosBOvK4VyD1WaNc","_score":null,"_source":{"@timestamp":"2023-11-09T23:09:44.201224634Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:09:44.200+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1049","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571384201]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7UNZtosBOvK4VyD1WaNc","_score":null,"_source":{"@timestamp":"2023-11-09T23:09:47.201749641Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:09:47.201+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1050","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571387201]},{"_index":"continuous-tests-pods-2023.11.09","_id":"p0NZtosBOvK4VyD1cb1z","_score":null,"_source":{"@timestamp":"2023-11-09T23:09:50.203184708Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:09:50.202+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1051","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571390203]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qENZtosBOvK4VyD1cb1z","_score":null,"_source":{"@timestamp":"2023-11-09T23:09:53.204084180Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:09:53.203+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1052","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571393204]},{"_index":"continuous-tests-pods-2023.11.09","_id":"h0NZtosBOvK4VyD1idaM","_score":null,"_source":{"@timestamp":"2023-11-09T23:09:56.205014358Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:09:56.204+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1053","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571396205]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iENZtosBOvK4VyD1idaM","_score":null,"_source":{"@timestamp":"2023-11-09T23:09:59.210815754Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:09:59.210+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1054","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571399210]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iUNZtosBOvK4VyD1idaM","_score":null,"_source":{"@timestamp":"2023-11-09T23:10:02.211355631Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:10:02.210+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1055","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571402211]},{"_index":"continuous-tests-pods-2023.11.09","_id":"a0RZtosBOvK4VyD1qQWt","_score":null,"_source":{"@timestamp":"2023-11-09T23:10:05.212494653Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:10:05.212+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1056","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571405212]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bERZtosBOvK4VyD1qQWt","_score":null,"_source":{"@timestamp":"2023-11-09T23:10:08.214630466Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:10:08.213+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1057","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571408214]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nERZtosBOvK4VyD1wTvE","_score":null,"_source":{"@timestamp":"2023-11-09T23:10:11.214645978Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:10:11.214+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1058","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571411214]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nURZtosBOvK4VyD1wTvE","_score":null,"_source":{"@timestamp":"2023-11-09T23:10:14.215091747Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:10:14.214+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1059","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571414215]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AURZtosBOvK4VyD12W3c","_score":null,"_source":{"@timestamp":"2023-11-09T23:10:17.215898106Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:10:17.215+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1060","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571417215]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AkRZtosBOvK4VyD12W3c","_score":null,"_source":{"@timestamp":"2023-11-09T23:10:20.217166595Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:10:20.216+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1061","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571420217]},{"_index":"continuous-tests-pods-2023.11.09","_id":"t0RZtosBOvK4VyD18Yj0","_score":null,"_source":{"@timestamp":"2023-11-09T23:10:23.217881193Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:10:23.217+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1062","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571423217]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uERZtosBOvK4VyD18Yj0","_score":null,"_source":{"@timestamp":"2023-11-09T23:10:26.218851778Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:10:26.218+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1063","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571426218]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WERatosBOvK4VyD1CssP","_score":null,"_source":{"@timestamp":"2023-11-09T23:10:29.219772853Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:10:29.219+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1064","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571429219]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WURatosBOvK4VyD1CssP","_score":null,"_source":{"@timestamp":"2023-11-09T23:10:32.225904690Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:10:32.225+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1065","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571432225]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZUVatosBOvK4VyD1Ih0r","_score":null,"_source":{"@timestamp":"2023-11-09T23:10:35.227078454Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:10:35.226+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1066","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571435227]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZkVatosBOvK4VyD1Ih0r","_score":null,"_source":{"@timestamp":"2023-11-09T23:10:38.228491974Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:10:38.228+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1067","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571438228]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KUVatosBOvK4VyD1NmhD","_score":null,"_source":{"@timestamp":"2023-11-09T23:10:41.230014676Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:10:41.229+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1068","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571441230]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KkVatosBOvK4VyD1NmhD","_score":null,"_source":{"@timestamp":"2023-11-09T23:10:44.230169468Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:10:44.229+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1069","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571444230]},{"_index":"continuous-tests-pods-2023.11.09","_id":"d0VatosBOvK4VyD1TrZX","_score":null,"_source":{"@timestamp":"2023-11-09T23:10:47.230769256Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:10:47.230+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1070","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571447230]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eEVatosBOvK4VyD1TrZX","_score":null,"_source":{"@timestamp":"2023-11-09T23:10:50.231613662Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:10:50.231+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1071","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571450231]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lUZatosBOvK4VyD1Zgls","_score":null,"_source":{"@timestamp":"2023-11-09T23:10:53.232168460Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:10:53.231+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1072","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571453232]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lkZatosBOvK4VyD1Zgls","_score":null,"_source":{"@timestamp":"2023-11-09T23:10:56.233311441Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:10:56.233+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1073","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571456233]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BEZatosBOvK4VyD1fi2E","_score":null,"_source":{"@timestamp":"2023-11-09T23:10:59.234228367Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:10:59.233+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1074","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571459234]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BUZatosBOvK4VyD1fi2E","_score":null,"_source":{"@timestamp":"2023-11-09T23:11:02.235366057Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:11:02.234+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1075","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571462235]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pkZatosBOvK4VyD1lkij","_score":null,"_source":{"@timestamp":"2023-11-09T23:11:05.236028944Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:11:05.235+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1076","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571465236]},{"_index":"continuous-tests-pods-2023.11.09","_id":"p0ZatosBOvK4VyD1lkij","_score":null,"_source":{"@timestamp":"2023-11-09T23:11:08.236942480Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:11:08.236+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1077","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571468236]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uUZatosBOvK4VyD1rmK7","_score":null,"_source":{"@timestamp":"2023-11-09T23:11:11.242560640Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:11:11.242+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1078","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571471242]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ukZatosBOvK4VyD1rmK7","_score":null,"_source":{"@timestamp":"2023-11-09T23:11:14.243608116Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:11:14.243+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1079","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571474243]},{"_index":"continuous-tests-pods-2023.11.09","_id":"u0ZatosBOvK4VyD1rmK7","_score":null,"_source":{"@timestamp":"2023-11-09T23:11:17.244308723Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:11:17.244+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1080","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571477244]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-0ZatosBOvK4VyD1zo7g","_score":null,"_source":{"@timestamp":"2023-11-09T23:11:20.245248551Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:11:20.244+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1081","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571480245]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_EZatosBOvK4VyD1zo7g","_score":null,"_source":{"@timestamp":"2023-11-09T23:11:23.246096315Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:11:23.245+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1082","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571483246]},{"_index":"continuous-tests-pods-2023.11.09","_id":"h0ZatosBOvK4VyD159MC","_score":null,"_source":{"@timestamp":"2023-11-09T23:11:26.246652402Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:11:26.246+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1083","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571486246]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iEZatosBOvK4VyD159MC","_score":null,"_source":{"@timestamp":"2023-11-09T23:11:29.247972167Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:11:29.247+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1084","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571489247]},{"_index":"continuous-tests-pods-2023.11.09","_id":"X0datosBOvK4VyD1_xgS","_score":null,"_source":{"@timestamp":"2023-11-09T23:11:32.248922390Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:11:32.248+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1085","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571492248]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YEdatosBOvK4VyD1_xgS","_score":null,"_source":{"@timestamp":"2023-11-09T23:11:35.249820128Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:11:35.249+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1086","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571495249]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1UdbtosBOvK4VyD1F1sm","_score":null,"_source":{"@timestamp":"2023-11-09T23:11:38.250435167Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:11:38.250+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1087","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571498250]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1kdbtosBOvK4VyD1F1sm","_score":null,"_source":{"@timestamp":"2023-11-09T23:11:41.251189526Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:11:41.250+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1088","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571501251]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xkdbtosBOvK4VyD1L5Q6","_score":null,"_source":{"@timestamp":"2023-11-09T23:11:44.252687808Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:11:44.252+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1089","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571504252]},{"_index":"continuous-tests-pods-2023.11.09","_id":"x0dbtosBOvK4VyD1L5Q6","_score":null,"_source":{"@timestamp":"2023-11-09T23:11:47.253186082Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:11:47.252+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1090","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571507253]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CUdbtosBOvK4VyD1R7NU","_score":null,"_source":{"@timestamp":"2023-11-09T23:11:50.258026110Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:11:50.257+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1091","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571510258]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CkdbtosBOvK4VyD1R7NU","_score":null,"_source":{"@timestamp":"2023-11-09T23:11:53.258828162Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:11:53.258+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1092","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571513258]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XkdbtosBOvK4VyD1W8pt","_score":null,"_source":{"@timestamp":"2023-11-09T23:11:56.260217806Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:11:56.259+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1093","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571516260]},{"_index":"continuous-tests-pods-2023.11.09","_id":"X0dbtosBOvK4VyD1W8pt","_score":null,"_source":{"@timestamp":"2023-11-09T23:11:59.261037961Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:11:59.260+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1094","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571519261]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wEdbtosBOvK4VyD1c-SJ","_score":null,"_source":{"@timestamp":"2023-11-09T23:12:02.261839988Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:12:02.261+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1095","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571522261]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wUdbtosBOvK4VyD1c-SJ","_score":null,"_source":{"@timestamp":"2023-11-09T23:12:05.262648359Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:12:05.262+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1096","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571525262]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TkhbtosBOvK4VyD1ixye","_score":null,"_source":{"@timestamp":"2023-11-09T23:12:08.263549978Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:12:08.263+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1097","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571528263]},{"_index":"continuous-tests-pods-2023.11.09","_id":"T0hbtosBOvK4VyD1ixye","_score":null,"_source":{"@timestamp":"2023-11-09T23:12:11.264806085Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:12:11.264+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1098","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571531264]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9EhbtosBOvK4VyD1o1Wy","_score":null,"_source":{"@timestamp":"2023-11-09T23:12:14.266116018Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:12:14.265+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1099","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571534266]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9UhbtosBOvK4VyD1o1Wy","_score":null,"_source":{"@timestamp":"2023-11-09T23:12:17.266721470Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:12:17.266+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1100","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571537266]},{"_index":"continuous-tests-pods-2023.11.09","_id":"S0hbtosBOvK4VyD1u3DJ","_score":null,"_source":{"@timestamp":"2023-11-09T23:12:20.267981707Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:12:20.267+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1101","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571540267]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TEhbtosBOvK4VyD1u3DJ","_score":null,"_source":{"@timestamp":"2023-11-09T23:12:23.269334841Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:12:23.268+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1102","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571543269]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ckhbtosBOvK4VyD105vg","_score":null,"_source":{"@timestamp":"2023-11-09T23:12:26.270035588Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:12:26.269+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1103","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571546270]},{"_index":"continuous-tests-pods-2023.11.09","_id":"c0hbtosBOvK4VyD105vg","_score":null,"_source":{"@timestamp":"2023-11-09T23:12:29.270652484Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:12:29.270+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1104","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571549270]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kUhbtosBOvK4VyD15-r4","_score":null,"_source":{"@timestamp":"2023-11-09T23:12:32.272123852Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:12:32.271+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1105","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571552272]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kkhbtosBOvK4VyD15-r4","_score":null,"_source":{"@timestamp":"2023-11-09T23:12:35.273565931Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:12:35.273+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1106","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571555273]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PklctosBOvK4VyD1AEEP","_score":null,"_source":{"@timestamp":"2023-11-09T23:12:38.279255015Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:12:38.278+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1107","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571558279]},{"_index":"continuous-tests-pods-2023.11.09","_id":"P0lctosBOvK4VyD1AEEP","_score":null,"_source":{"@timestamp":"2023-11-09T23:12:41.279846499Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:12:41.279+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1108","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571561279]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9ElctosBOvK4VyD1GJYo","_score":null,"_source":{"@timestamp":"2023-11-09T23:12:44.281486560Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:12:44.281+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1109","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571564281]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9UlctosBOvK4VyD1GJYo","_score":null,"_source":{"@timestamp":"2023-11-09T23:12:47.282197744Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:12:47.281+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1110","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571567282]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VklctosBOvK4VyD1MOpN","_score":null,"_source":{"@timestamp":"2023-11-09T23:12:50.284137088Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:12:50.283+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1111","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571570284]},{"_index":"continuous-tests-pods-2023.11.09","_id":"V0lctosBOvK4VyD1MOpN","_score":null,"_source":{"@timestamp":"2023-11-09T23:12:53.285806806Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:12:53.285+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1112","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571573285]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VUpctosBOvK4VyD1SBJs","_score":null,"_source":{"@timestamp":"2023-11-09T23:12:56.286813965Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:12:56.286+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1113","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571576286]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VkpctosBOvK4VyD1SBJs","_score":null,"_source":{"@timestamp":"2023-11-09T23:12:59.287445833Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:12:59.287+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1114","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571579287]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EUpctosBOvK4VyD1YC6L","_score":null,"_source":{"@timestamp":"2023-11-09T23:13:02.288750828Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:13:02.288+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1115","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571582288]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EkpctosBOvK4VyD1YC6L","_score":null,"_source":{"@timestamp":"2023-11-09T23:13:03.870776312Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:13:03.870+00:00 Cleaning up expired records tid=1 + count=1116","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571583870]},{"_index":"continuous-tests-pods-2023.11.09","_id":"E0pctosBOvK4VyD1YC6L","_score":null,"_source":{"@timestamp":"2023-11-09T23:13:03.870969032Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:13:03.870+00:00 Cleaned up expired records tid=1 + size=0 count=1117","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571583870]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FEpctosBOvK4VyD1YC6L","_score":null,"_source":{"@timestamp":"2023-11-09T23:13:03.870993305Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:13:03.870+00:00 Cleaning up query iterator tid=1 + count=1118","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571583870]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FUpctosBOvK4VyD1YC6L","_score":null,"_source":{"@timestamp":"2023-11-09T23:13:03.870998244Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:13:03.870+00:00 Cleaning up orphaned records tid=1 + count=1119","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571583870]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FkpctosBOvK4VyD1YC6L","_score":null,"_source":{"@timestamp":"2023-11-09T23:13:03.871031828Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:13:03.870+00:00 Cleaning up query iterator tid=1 + count=1120","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571583871]},{"_index":"continuous-tests-pods-2023.11.09","_id":"F0pctosBOvK4VyD1YC6L","_score":null,"_source":{"@timestamp":"2023-11-09T23:13:05.289733152Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:13:05.289+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1121","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571585289]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zUpctosBOvK4VyD1dEOj","_score":null,"_source":{"@timestamp":"2023-11-09T23:13:08.290712760Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:13:08.290+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1122","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571588290]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zkpctosBOvK4VyD1dEOj","_score":null,"_source":{"@timestamp":"2023-11-09T23:13:11.291855328Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:13:11.291+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1123","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571591291]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MkpctosBOvK4VyD1jGK_","_score":null,"_source":{"@timestamp":"2023-11-09T23:13:14.293283410Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:13:14.292+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1124","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571594293]},{"_index":"continuous-tests-pods-2023.11.09","_id":"M0pctosBOvK4VyD1jGK_","_score":null,"_source":{"@timestamp":"2023-11-09T23:13:17.294030145Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:13:17.293+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1125","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571597294]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SEpctosBOvK4VyD1pIPY","_score":null,"_source":{"@timestamp":"2023-11-09T23:13:20.295127478Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:13:20.294+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1126","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571600295]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SUpctosBOvK4VyD1pIPY","_score":null,"_source":{"@timestamp":"2023-11-09T23:13:23.296453825Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:13:23.296+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1127","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571603296]},{"_index":"continuous-tests-pods-2023.11.09","_id":"U0pctosBOvK4VyD1vMjt","_score":null,"_source":{"@timestamp":"2023-11-09T23:13:26.297129182Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:13:26.296+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1128","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571606297]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VEpctosBOvK4VyD1vMjt","_score":null,"_source":{"@timestamp":"2023-11-09T23:13:29.297958997Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:13:29.297+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1129","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571609297]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jEtctosBOvK4VyD11Q4O","_score":null,"_source":{"@timestamp":"2023-11-09T23:13:32.301680790Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:13:32.298+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1130","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571612301]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jUtctosBOvK4VyD11Q4O","_score":null,"_source":{"@timestamp":"2023-11-09T23:13:35.299655068Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:13:35.299+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1131","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571615299]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iktctosBOvK4VyD17U4p","_score":null,"_source":{"@timestamp":"2023-11-09T23:13:38.300593861Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:13:38.300+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1132","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571618300]},{"_index":"continuous-tests-pods-2023.11.09","_id":"i0tctosBOvK4VyD17U4p","_score":null,"_source":{"@timestamp":"2023-11-09T23:13:41.302033724Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:13:41.301+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1133","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571621302]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jEtctosBOvK4VyD17U4p","_score":null,"_source":{"@timestamp":"2023-11-09T23:13:44.303272314Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:13:44.302+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1134","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571624303]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uktdtosBOvK4VyD1DYlN","_score":null,"_source":{"@timestamp":"2023-11-09T23:13:47.304518892Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:13:47.304+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1135","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571627304]},{"_index":"continuous-tests-pods-2023.11.09","_id":"u0tdtosBOvK4VyD1DYlN","_score":null,"_source":{"@timestamp":"2023-11-09T23:13:50.305992045Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:13:50.305+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1136","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571630305]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EUtdtosBOvK4VyD1JbRl","_score":null,"_source":{"@timestamp":"2023-11-09T23:13:53.306915200Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:13:53.306+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1137","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571633306]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EktdtosBOvK4VyD1JbRl","_score":null,"_source":{"@timestamp":"2023-11-09T23:13:56.308011221Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:13:56.307+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1138","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571636308]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dktdtosBOvK4VyD1Pc6A","_score":null,"_source":{"@timestamp":"2023-11-09T23:13:59.308547176Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:13:59.308+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1139","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571639308]},{"_index":"continuous-tests-pods-2023.11.09","_id":"d0tdtosBOvK4VyD1Pc6A","_score":null,"_source":{"@timestamp":"2023-11-09T23:14:02.309936862Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:14:02.309+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1140","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571642309]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NEtdtosBOvK4VyD1VfOa","_score":null,"_source":{"@timestamp":"2023-11-09T23:14:05.310420192Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:14:05.310+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1141","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571645310]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NUtdtosBOvK4VyD1VfOa","_score":null,"_source":{"@timestamp":"2023-11-09T23:14:08.311265414Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:14:08.310+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1142","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571648311]},{"_index":"continuous-tests-pods-2023.11.09","_id":"F0xdtosBOvK4VyD1bSuw","_score":null,"_source":{"@timestamp":"2023-11-09T23:14:11.316598475Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:14:11.316+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1143","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571651316]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GExdtosBOvK4VyD1bSuw","_score":null,"_source":{"@timestamp":"2023-11-09T23:14:14.317350030Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:14:14.316+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1144","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571654317]},{"_index":"continuous-tests-pods-2023.11.09","_id":"v0xdtosBOvK4VyD1hVrG","_score":null,"_source":{"@timestamp":"2023-11-09T23:14:17.318222721Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:14:17.317+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1145","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571657318]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wExdtosBOvK4VyD1hVrG","_score":null,"_source":{"@timestamp":"2023-11-09T23:14:20.319344779Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:14:20.319+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1146","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571660319]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-ExdtosBOvK4VyD1mXHe","_score":null,"_source":{"@timestamp":"2023-11-09T23:14:23.320677691Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:14:23.320+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1147","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571663320]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-UxdtosBOvK4VyD1mXHe","_score":null,"_source":{"@timestamp":"2023-11-09T23:14:26.321627036Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:14:26.321+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1148","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571666321]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rUxdtosBOvK4VyD1sZv5","_score":null,"_source":{"@timestamp":"2023-11-09T23:14:29.322825431Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:14:29.322+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1149","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571669322]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rkxdtosBOvK4VyD1sZv5","_score":null,"_source":{"@timestamp":"2023-11-09T23:14:32.324400043Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:14:32.323+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1150","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571672324]},{"_index":"continuous-tests-pods-2023.11.09","_id":"C0xdtosBOvK4VyD1yvMQ","_score":null,"_source":{"@timestamp":"2023-11-09T23:14:35.325986713Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:14:35.325+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1151","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571675325]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DExdtosBOvK4VyD1yvMQ","_score":null,"_source":{"@timestamp":"2023-11-09T23:14:38.331580172Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:14:38.331+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1152","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571678331]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0k1dtosBOvK4VyD14kUm","_score":null,"_source":{"@timestamp":"2023-11-09T23:14:41.333166274Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:14:41.332+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1153","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571681333]},{"_index":"continuous-tests-pods-2023.11.09","_id":"001dtosBOvK4VyD14kUm","_score":null,"_source":{"@timestamp":"2023-11-09T23:14:44.334360245Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:14:44.334+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1154","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571684334]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pk1dtosBOvK4VyD1-pk9","_score":null,"_source":{"@timestamp":"2023-11-09T23:14:47.335839389Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:14:47.335+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1155","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571687335]},{"_index":"continuous-tests-pods-2023.11.09","_id":"p01dtosBOvK4VyD1-pk9","_score":null,"_source":{"@timestamp":"2023-11-09T23:14:50.337025926Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:14:50.336+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1156","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571690337]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BU1etosBOvK4VyD1EupT","_score":null,"_source":{"@timestamp":"2023-11-09T23:14:53.337923390Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:14:53.337+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1157","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571693337]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Bk1etosBOvK4VyD1EupT","_score":null,"_source":{"@timestamp":"2023-11-09T23:14:56.338797376Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:14:56.338+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1158","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571696338]},{"_index":"continuous-tests-pods-2023.11.09","_id":"B01etosBOvK4VyD1EupT","_score":null,"_source":{"@timestamp":"2023-11-09T23:14:59.339988126Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:14:59.339+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1159","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571699339]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Nk5etosBOvK4VyD1MiB6","_score":null,"_source":{"@timestamp":"2023-11-09T23:15:02.340690232Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:15:02.340+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1160","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571702340]},{"_index":"continuous-tests-pods-2023.11.09","_id":"N05etosBOvK4VyD1MiB6","_score":null,"_source":{"@timestamp":"2023-11-09T23:15:05.341277087Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:15:05.340+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1161","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571705341]},{"_index":"continuous-tests-pods-2023.11.09","_id":"b05etosBOvK4VyD1SjqT","_score":null,"_source":{"@timestamp":"2023-11-09T23:15:08.342137921Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:15:08.341+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1162","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571708342]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cE5etosBOvK4VyD1SjqT","_score":null,"_source":{"@timestamp":"2023-11-09T23:15:11.342434558Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:15:11.342+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1163","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571711342]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZU5etosBOvK4VyD1Yleu","_score":null,"_source":{"@timestamp":"2023-11-09T23:15:14.343329598Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:15:14.342+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1164","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571714343]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Zk5etosBOvK4VyD1Yleu","_score":null,"_source":{"@timestamp":"2023-11-09T23:15:17.343872377Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:15:17.343+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1165","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571717343]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QE5etosBOvK4VyD1enfI","_score":null,"_source":{"@timestamp":"2023-11-09T23:15:20.345041354Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:15:20.344+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1166","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571720345]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QU5etosBOvK4VyD1enfI","_score":null,"_source":{"@timestamp":"2023-11-09T23:15:23.346575101Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:15:23.346+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1167","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571723346]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3U5etosBOvK4VyD1kr3k","_score":null,"_source":{"@timestamp":"2023-11-09T23:15:26.347777553Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:15:26.347+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1168","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571726347]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3k5etosBOvK4VyD1kr3k","_score":null,"_source":{"@timestamp":"2023-11-09T23:15:29.348616430Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:15:29.348+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1169","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571729348]},{"_index":"continuous-tests-pods-2023.11.09","_id":"n09etosBOvK4VyD1qgj1","_score":null,"_source":{"@timestamp":"2023-11-09T23:15:32.354173129Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:15:32.353+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1170","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571732354]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oE9etosBOvK4VyD1qgj1","_score":null,"_source":{"@timestamp":"2023-11-09T23:15:35.355073951Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:15:35.354+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1171","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571735355]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_U9etosBOvK4VyD1v0AN","_score":null,"_source":{"@timestamp":"2023-11-09T23:15:38.356205705Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:15:38.355+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1172","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571738356]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_k9etosBOvK4VyD1v0AN","_score":null,"_source":{"@timestamp":"2023-11-09T23:15:41.357687745Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:15:41.357+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1173","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571741357]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hU9etosBOvK4VyD113gm","_score":null,"_source":{"@timestamp":"2023-11-09T23:15:44.358744464Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:15:44.358+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1174","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571744358]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hk9etosBOvK4VyD113gm","_score":null,"_source":{"@timestamp":"2023-11-09T23:15:47.359437121Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:15:47.359+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1175","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571747359]},{"_index":"continuous-tests-pods-2023.11.09","_id":"w09etosBOvK4VyD175k_","_score":null,"_source":{"@timestamp":"2023-11-09T23:15:50.360820945Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:15:50.360+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1176","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571750360]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xE9etosBOvK4VyD175k_","_score":null,"_source":{"@timestamp":"2023-11-09T23:15:53.361581571Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:15:53.361+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1177","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571753361]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8E9ftosBOvK4VyD1B7NX","_score":null,"_source":{"@timestamp":"2023-11-09T23:15:56.363074035Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:15:56.362+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1178","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571756363]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8U9ftosBOvK4VyD1B7NX","_score":null,"_source":{"@timestamp":"2023-11-09T23:15:59.364501503Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:15:59.364+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1179","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571759364]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ek9ftosBOvK4VyD1H9CA","_score":null,"_source":{"@timestamp":"2023-11-09T23:16:02.365807449Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:16:02.365+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1180","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571762365]},{"_index":"continuous-tests-pods-2023.11.09","_id":"E09ftosBOvK4VyD1H9CA","_score":null,"_source":{"@timestamp":"2023-11-09T23:16:05.366134956Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:16:05.365+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1181","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571765366]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9E9ftosBOvK4VyD1N_6L","_score":null,"_source":{"@timestamp":"2023-11-09T23:16:08.367448040Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:16:08.367+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1182","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571768367]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9U9ftosBOvK4VyD1N_6L","_score":null,"_source":{"@timestamp":"2023-11-09T23:16:11.368073582Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:16:11.367+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1183","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571771368]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3VBftosBOvK4VyD1Syuh","_score":null,"_source":{"@timestamp":"2023-11-09T23:16:14.369607597Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:16:14.369+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1184","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571774369]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3lBftosBOvK4VyD1Syuh","_score":null,"_source":{"@timestamp":"2023-11-09T23:16:17.370465818Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:16:17.370+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1185","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571777370]},{"_index":"continuous-tests-pods-2023.11.09","_id":"y1BftosBOvK4VyD1Y1i6","_score":null,"_source":{"@timestamp":"2023-11-09T23:16:20.371788687Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:16:20.371+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1186","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571780371]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zFBftosBOvK4VyD1Y1i6","_score":null,"_source":{"@timestamp":"2023-11-09T23:16:23.373495557Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:16:23.373+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1187","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571783373]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MFBftosBOvK4VyD1e3TV","_score":null,"_source":{"@timestamp":"2023-11-09T23:16:26.374238438Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:16:26.373+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1188","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571786374]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MVBftosBOvK4VyD1e3TV","_score":null,"_source":{"@timestamp":"2023-11-09T23:16:29.375691185Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:16:29.375+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1189","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571789375]},{"_index":"continuous-tests-pods-2023.11.09","_id":"plBftosBOvK4VyD1k8Dv","_score":null,"_source":{"@timestamp":"2023-11-09T23:16:32.376367693Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:16:32.376+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1190","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571792376]},{"_index":"continuous-tests-pods-2023.11.09","_id":"p1BftosBOvK4VyD1k8Dv","_score":null,"_source":{"@timestamp":"2023-11-09T23:16:35.377874613Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:16:35.377+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1191","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571795377]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FFFftosBOvK4VyD1rBcF","_score":null,"_source":{"@timestamp":"2023-11-09T23:16:38.378678071Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:16:38.378+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1192","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571798378]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FVFftosBOvK4VyD1rBcF","_score":null,"_source":{"@timestamp":"2023-11-09T23:16:41.380261308Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:16:41.379+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1193","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571801380]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XVFftosBOvK4VyD1xGwd","_score":null,"_source":{"@timestamp":"2023-11-09T23:16:44.380996172Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:16:44.380+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1194","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571804380]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XlFftosBOvK4VyD1xGwd","_score":null,"_source":{"@timestamp":"2023-11-09T23:16:47.386643862Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:16:47.386+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1195","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571807386]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZFFftosBOvK4VyD12LI1","_score":null,"_source":{"@timestamp":"2023-11-09T23:16:50.387438669Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:16:50.387+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1196","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571810387]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZVFftosBOvK4VyD12LI1","_score":null,"_source":{"@timestamp":"2023-11-09T23:16:53.388086881Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:16:53.387+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1197","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571813388]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LVFftosBOvK4VyD18PtM","_score":null,"_source":{"@timestamp":"2023-11-09T23:16:56.389156612Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:16:56.388+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1198","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571816389]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LlFftosBOvK4VyD18PtM","_score":null,"_source":{"@timestamp":"2023-11-09T23:16:59.389765110Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:16:59.389+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1199","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571819389]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LFJgtosBOvK4VyD1CBVk","_score":null,"_source":{"@timestamp":"2023-11-09T23:17:02.390722177Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:17:02.390+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1200","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571822390]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LVJgtosBOvK4VyD1CBVk","_score":null,"_source":{"@timestamp":"2023-11-09T23:17:05.391421296Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:17:05.391+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1201","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571825391]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6lJgtosBOvK4VyD1IDB_","_score":null,"_source":{"@timestamp":"2023-11-09T23:17:08.393155983Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:17:08.392+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1202","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571828393]},{"_index":"continuous-tests-pods-2023.11.09","_id":"61JgtosBOvK4VyD1IDB_","_score":null,"_source":{"@timestamp":"2023-11-09T23:17:11.394099038Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:17:11.393+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1203","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571831394]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0lJgtosBOvK4VyD1OEyY","_score":null,"_source":{"@timestamp":"2023-11-09T23:17:14.395546716Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:17:14.395+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1204","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571834395]},{"_index":"continuous-tests-pods-2023.11.09","_id":"01JgtosBOvK4VyD1OEyY","_score":null,"_source":{"@timestamp":"2023-11-09T23:17:17.396741732Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:17:17.396+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1205","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571837396]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kVJgtosBOvK4VyD1UG-u","_score":null,"_source":{"@timestamp":"2023-11-09T23:17:20.402140557Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:17:20.401+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1206","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571840402]},{"_index":"continuous-tests-pods-2023.11.09","_id":"klJgtosBOvK4VyD1UG-u","_score":null,"_source":{"@timestamp":"2023-11-09T23:17:23.403205455Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:17:23.402+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1207","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571843403]},{"_index":"continuous-tests-pods-2023.11.09","_id":"k1JgtosBOvK4VyD1UG-u","_score":null,"_source":{"@timestamp":"2023-11-09T23:17:26.404176380Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:17:26.403+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1208","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571846404]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_VJgtosBOvK4VyD1cMTW","_score":null,"_source":{"@timestamp":"2023-11-09T23:17:29.404909117Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:17:29.404+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1209","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571849404]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_lJgtosBOvK4VyD1cMTW","_score":null,"_source":{"@timestamp":"2023-11-09T23:17:32.406116799Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:17:32.405+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1210","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571852406]},{"_index":"continuous-tests-pods-2023.11.09","_id":"alNgtosBOvK4VyD1iCHu","_score":null,"_source":{"@timestamp":"2023-11-09T23:17:35.406918270Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:17:35.406+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1211","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571855406]},{"_index":"continuous-tests-pods-2023.11.09","_id":"a1NgtosBOvK4VyD1iCHu","_score":null,"_source":{"@timestamp":"2023-11-09T23:17:38.408346442Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:17:38.408+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1212","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571858408]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hlNgtosBOvK4VyD1oUsH","_score":null,"_source":{"@timestamp":"2023-11-09T23:17:41.409843443Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:17:41.409+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1213","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571861409]},{"_index":"continuous-tests-pods-2023.11.09","_id":"h1NgtosBOvK4VyD1oUsH","_score":null,"_source":{"@timestamp":"2023-11-09T23:17:44.411132034Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:17:44.410+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1214","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571864411]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_FNgtosBOvK4VyD1uX8i","_score":null,"_source":{"@timestamp":"2023-11-09T23:17:47.411979921Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:17:47.411+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1215","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571867411]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_VNgtosBOvK4VyD1uX8i","_score":null,"_source":{"@timestamp":"2023-11-09T23:17:50.413700219Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:17:50.413+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1216","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571870413]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KVNgtosBOvK4VyD10Zc4","_score":null,"_source":{"@timestamp":"2023-11-09T23:17:53.414998870Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:17:53.414+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1217","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571873414]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KlNgtosBOvK4VyD10Zc4","_score":null,"_source":{"@timestamp":"2023-11-09T23:17:56.415753538Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:17:56.415+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1218","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571876415]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5lNgtosBOvK4VyD16bJP","_score":null,"_source":{"@timestamp":"2023-11-09T23:17:59.417496804Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:17:59.417+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1219","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571879417]},{"_index":"continuous-tests-pods-2023.11.09","_id":"51NgtosBOvK4VyD16bJP","_score":null,"_source":{"@timestamp":"2023-11-09T23:18:02.418256632Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:18:02.417+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1220","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571882418]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6FNgtosBOvK4VyD16bJP","_score":null,"_source":{"@timestamp":"2023-11-09T23:18:03.873181702Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:18:03.872+00:00 Cleaning up expired records tid=1 + count=1221","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571883873]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6VNgtosBOvK4VyD16bJP","_score":null,"_source":{"@timestamp":"2023-11-09T23:18:03.873230185Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:18:03.872+00:00 Cleaned up expired records tid=1 + size=0 count=1222","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571883873]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6lNgtosBOvK4VyD16bJP","_score":null,"_source":{"@timestamp":"2023-11-09T23:18:03.873237679Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:18:03.872+00:00 Cleaning up query iterator tid=1 + count=1223","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571883873]},{"_index":"continuous-tests-pods-2023.11.09","_id":"61NgtosBOvK4VyD16bJP","_score":null,"_source":{"@timestamp":"2023-11-09T23:18:03.873243583Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:18:03.872+00:00 Cleaning up orphaned records tid=1 + count=1224","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571883873]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7FNgtosBOvK4VyD16bJP","_score":null,"_source":{"@timestamp":"2023-11-09T23:18:03.873247902Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:18:03.872+00:00 Cleaning up query iterator tid=1 + count=1225","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571883873]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7VNgtosBOvK4VyD16bJP","_score":null,"_source":{"@timestamp":"2023-11-09T23:18:05.418960896Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:18:05.418+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1226","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571885418]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1VNhtosBOvK4VyD1Ce5-","_score":null,"_source":{"@timestamp":"2023-11-09T23:18:08.420353643Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:18:08.420+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1227","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571888420]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1lNhtosBOvK4VyD1Ce5-","_score":null,"_source":{"@timestamp":"2023-11-09T23:18:11.421240549Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:18:11.420+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1228","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571891421]},{"_index":"continuous-tests-pods-2023.11.09","_id":"llRhtosBOvK4VyD1ISaZ","_score":null,"_source":{"@timestamp":"2023-11-09T23:18:14.428342167Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:18:14.428+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1229","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571894428]},{"_index":"continuous-tests-pods-2023.11.09","_id":"l1RhtosBOvK4VyD1ISaZ","_score":null,"_source":{"@timestamp":"2023-11-09T23:18:17.429482407Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:18:17.429+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1230","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571897429]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LVRhtosBOvK4VyD1OT-0","_score":null,"_source":{"@timestamp":"2023-11-09T23:18:20.430670520Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:18:20.430+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1231","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571900430]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LlRhtosBOvK4VyD1OT-0","_score":null,"_source":{"@timestamp":"2023-11-09T23:18:23.431583398Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:18:23.431+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1232","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571903431]},{"_index":"continuous-tests-pods-2023.11.09","_id":"51RhtosBOvK4VyD1UXDL","_score":null,"_source":{"@timestamp":"2023-11-09T23:18:26.432960806Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:18:26.432+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1233","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571906432]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6FRhtosBOvK4VyD1UXDL","_score":null,"_source":{"@timestamp":"2023-11-09T23:18:29.433726210Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:18:29.433+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1234","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571909433]},{"_index":"continuous-tests-pods-2023.11.09","_id":"O1RhtosBOvK4VyD1acPj","_score":null,"_source":{"@timestamp":"2023-11-09T23:18:32.434805035Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:18:32.434+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1235","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571912434]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PFRhtosBOvK4VyD1acPj","_score":null,"_source":{"@timestamp":"2023-11-09T23:18:35.436315973Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:18:35.436+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1236","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571915436]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PVRhtosBOvK4VyD1acPj","_score":null,"_source":{"@timestamp":"2023-11-09T23:18:38.437726955Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:18:38.437+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1237","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571918437]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5FVhtosBOvK4VyD1ijAD","_score":null,"_source":{"@timestamp":"2023-11-09T23:18:41.443386329Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:18:41.443+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1238","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571921443]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5VVhtosBOvK4VyD1ijAD","_score":null,"_source":{"@timestamp":"2023-11-09T23:18:44.444559427Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:18:44.444+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1239","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571924444]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OFVhtosBOvK4VyD1ooYd","_score":null,"_source":{"@timestamp":"2023-11-09T23:18:47.445172829Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:18:47.444+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1240","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571927445]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OVVhtosBOvK4VyD1ooYd","_score":null,"_source":{"@timestamp":"2023-11-09T23:18:50.446650641Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:18:50.446+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1241","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571930446]},{"_index":"continuous-tests-pods-2023.11.09","_id":"T1VhtosBOvK4VyD1urkx","_score":null,"_source":{"@timestamp":"2023-11-09T23:18:53.448088158Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:18:53.447+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1242","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571933448]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UFVhtosBOvK4VyD1urkx","_score":null,"_source":{"@timestamp":"2023-11-09T23:18:56.449050017Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:18:56.448+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1243","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571936449]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tVVhtosBOvK4VyD10tFK","_score":null,"_source":{"@timestamp":"2023-11-09T23:18:59.450407181Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:18:59.450+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1244","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571939450]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tlVhtosBOvK4VyD10tFK","_score":null,"_source":{"@timestamp":"2023-11-09T23:19:02.451554147Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:19:02.451+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1245","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571942451]},{"_index":"continuous-tests-pods-2023.11.09","_id":"11VhtosBOvK4VyD16uhn","_score":null,"_source":{"@timestamp":"2023-11-09T23:19:05.452217182Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:19:05.451+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1246","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571945452]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2FVhtosBOvK4VyD16uhn","_score":null,"_source":{"@timestamp":"2023-11-09T23:19:08.458362825Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:19:08.458+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1247","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571948458]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zFVitosBOvK4VyD1Av98","_score":null,"_source":{"@timestamp":"2023-11-09T23:19:11.459538190Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:19:11.459+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1248","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571951459]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zVVitosBOvK4VyD1Av98","_score":null,"_source":{"@timestamp":"2023-11-09T23:19:14.460211440Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:19:14.459+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1249","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571954460]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zlVitosBOvK4VyD1Av98","_score":null,"_source":{"@timestamp":"2023-11-09T23:19:17.461857258Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:19:17.461+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1250","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571957461]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HVZitosBOvK4VyD1IiKf","_score":null,"_source":{"@timestamp":"2023-11-09T23:19:20.463311662Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:19:20.463+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1251","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571960463]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HlZitosBOvK4VyD1IiKf","_score":null,"_source":{"@timestamp":"2023-11-09T23:19:23.464442663Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:19:23.464+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1252","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571963464]},{"_index":"continuous-tests-pods-2023.11.09","_id":"R1ZitosBOvK4VyD1Ol-4","_score":null,"_source":{"@timestamp":"2023-11-09T23:19:26.465931193Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:19:26.465+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1253","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571966465]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SFZitosBOvK4VyD1Ol-4","_score":null,"_source":{"@timestamp":"2023-11-09T23:19:29.466691900Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:19:29.466+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1254","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571969466]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UVZitosBOvK4VyD1UqHP","_score":null,"_source":{"@timestamp":"2023-11-09T23:19:32.467825191Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:19:32.467+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1255","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571972467]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UlZitosBOvK4VyD1UqHP","_score":null,"_source":{"@timestamp":"2023-11-09T23:19:35.468716682Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:19:35.468+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1256","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571975468]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OFZitosBOvK4VyD1auHo","_score":null,"_source":{"@timestamp":"2023-11-09T23:19:38.469748647Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:19:38.469+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1257","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571978469]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OVZitosBOvK4VyD1auHo","_score":null,"_source":{"@timestamp":"2023-11-09T23:19:41.471300711Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:19:41.470+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1258","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571981471]},{"_index":"continuous-tests-pods-2023.11.09","_id":"y1ditosBOvK4VyD1gxIC","_score":null,"_source":{"@timestamp":"2023-11-09T23:19:44.472179904Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:19:44.471+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1259","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571984472]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zFditosBOvK4VyD1gxIC","_score":null,"_source":{"@timestamp":"2023-11-09T23:19:47.472862160Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:19:47.472+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1260","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571987472]},{"_index":"continuous-tests-pods-2023.11.09","_id":"R1ditosBOvK4VyD1mzEc","_score":null,"_source":{"@timestamp":"2023-11-09T23:19:50.474544065Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:19:50.474+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1261","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571990474]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SFditosBOvK4VyD1mzEc","_score":null,"_source":{"@timestamp":"2023-11-09T23:19:53.475413964Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:19:53.475+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1262","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571993475]},{"_index":"continuous-tests-pods-2023.11.09","_id":"31ditosBOvK4VyD1r0M1","_score":null,"_source":{"@timestamp":"2023-11-09T23:19:56.481056323Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:19:56.480+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1263","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571996481]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4FditosBOvK4VyD1r0M1","_score":null,"_source":{"@timestamp":"2023-11-09T23:19:59.482351220Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:19:59.482+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1264","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699571999482]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dFditosBOvK4VyD1x1ZQ","_score":null,"_source":{"@timestamp":"2023-11-09T23:20:02.483669051Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:20:02.483+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1265","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572002483]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dVditosBOvK4VyD1x1ZQ","_score":null,"_source":{"@timestamp":"2023-11-09T23:20:05.484805276Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:20:05.484+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1266","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572005484]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dVditosBOvK4VyD134ln","_score":null,"_source":{"@timestamp":"2023-11-09T23:20:08.486362458Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:20:08.485+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1267","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572008486]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dlditosBOvK4VyD134ln","_score":null,"_source":{"@timestamp":"2023-11-09T23:20:11.487125415Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:20:11.486+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1268","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572011487]},{"_index":"continuous-tests-pods-2023.11.09","_id":"71ditosBOvK4VyD197-O","_score":null,"_source":{"@timestamp":"2023-11-09T23:20:14.487847978Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:20:14.487+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1269","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572014487]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8FditosBOvK4VyD197-O","_score":null,"_source":{"@timestamp":"2023-11-09T23:20:17.489319328Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:20:17.488+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1270","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572017489]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JFdjtosBOvK4VyD1D9iZ","_score":null,"_source":{"@timestamp":"2023-11-09T23:20:20.490758843Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:20:20.490+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1271","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572020490]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JVdjtosBOvK4VyD1D9iZ","_score":null,"_source":{"@timestamp":"2023-11-09T23:20:23.492250216Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:20:23.491+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1272","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572023492]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bFhjtosBOvK4VyD1JwWv","_score":null,"_source":{"@timestamp":"2023-11-09T23:20:26.493214185Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:20:26.492+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1273","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572026493]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bVhjtosBOvK4VyD1JwWv","_score":null,"_source":{"@timestamp":"2023-11-09T23:20:29.494151784Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:20:29.493+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1274","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572029494]},{"_index":"continuous-tests-pods-2023.11.09","_id":"blhjtosBOvK4VyD1JwWv","_score":null,"_source":{"@timestamp":"2023-11-09T23:20:32.495717952Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:20:32.495+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1275","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572032495]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WFhjtosBOvK4VyD1R3TW","_score":null,"_source":{"@timestamp":"2023-11-09T23:20:35.497431882Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:20:35.497+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1276","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572035497]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WVhjtosBOvK4VyD1R3TW","_score":null,"_source":{"@timestamp":"2023-11-09T23:20:38.498263759Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:20:38.497+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1277","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572038498]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uVhjtosBOvK4VyD1X8nt","_score":null,"_source":{"@timestamp":"2023-11-09T23:20:41.499375752Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:20:41.498+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1278","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572041499]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ulhjtosBOvK4VyD1X8nt","_score":null,"_source":{"@timestamp":"2023-11-09T23:20:44.500119339Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:20:44.499+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1279","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572044500]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tFljtosBOvK4VyD1eB0F","_score":null,"_source":{"@timestamp":"2023-11-09T23:20:47.501378852Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:20:47.500+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1280","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572047501]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tVljtosBOvK4VyD1eB0F","_score":null,"_source":{"@timestamp":"2023-11-09T23:20:50.502971229Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:20:50.502+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1281","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572050502]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IFljtosBOvK4VyD1kFQg","_score":null,"_source":{"@timestamp":"2023-11-09T23:20:53.503482423Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:20:53.503+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1282","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572053503]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IVljtosBOvK4VyD1kFQg","_score":null,"_source":{"@timestamp":"2023-11-09T23:20:56.505155274Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:20:56.504+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1283","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572056505]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ylljtosBOvK4VyD1qGw4","_score":null,"_source":{"@timestamp":"2023-11-09T23:20:59.506684055Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:20:59.506+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1284","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572059506]},{"_index":"continuous-tests-pods-2023.11.09","_id":"y1ljtosBOvK4VyD1qGw4","_score":null,"_source":{"@timestamp":"2023-11-09T23:21:02.508064885Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:21:02.507+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1285","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572062508]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PlljtosBOvK4VyD1wIJR","_score":null,"_source":{"@timestamp":"2023-11-09T23:21:05.509413290Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:21:05.508+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1286","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572065509]},{"_index":"continuous-tests-pods-2023.11.09","_id":"P1ljtosBOvK4VyD1wIJR","_score":null,"_source":{"@timestamp":"2023-11-09T23:21:08.510352008Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:21:08.509+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1287","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572068510]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CVljtosBOvK4VyD11JZl","_score":null,"_source":{"@timestamp":"2023-11-09T23:21:11.511946890Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:21:11.511+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1288","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572071511]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ClljtosBOvK4VyD11JZl","_score":null,"_source":{"@timestamp":"2023-11-09T23:21:14.512281653Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:21:14.511+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1289","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572074512]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qFljtosBOvK4VyD17Kt9","_score":null,"_source":{"@timestamp":"2023-11-09T23:21:17.513832768Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:21:17.513+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1290","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572077513]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qVljtosBOvK4VyD17Kt9","_score":null,"_source":{"@timestamp":"2023-11-09T23:21:20.514908624Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:21:20.514+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1291","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572080514]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3FlktosBOvK4VyD1BNiX","_score":null,"_source":{"@timestamp":"2023-11-09T23:21:23.516755513Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:21:23.516+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1292","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572083516]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3VlktosBOvK4VyD1BNiX","_score":null,"_source":{"@timestamp":"2023-11-09T23:21:26.518096201Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:21:26.517+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1293","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572086518]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HVpktosBOvK4VyD1HA2x","_score":null,"_source":{"@timestamp":"2023-11-09T23:21:29.518780431Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:21:29.518+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1294","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572089518]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HlpktosBOvK4VyD1HA2x","_score":null,"_source":{"@timestamp":"2023-11-09T23:21:32.519594950Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:21:32.519+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1295","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572092519]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GFpktosBOvK4VyD1NGrK","_score":null,"_source":{"@timestamp":"2023-11-09T23:21:35.521209610Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:21:35.520+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1296","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572095521]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GVpktosBOvK4VyD1NGrK","_score":null,"_source":{"@timestamp":"2023-11-09T23:21:38.522473365Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:21:38.522+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1297","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572098522]},{"_index":"continuous-tests-pods-2023.11.09","_id":"U1pktosBOvK4VyD1TJDk","_score":null,"_source":{"@timestamp":"2023-11-09T23:21:41.523760862Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:21:41.523+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1298","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572101523]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VFpktosBOvK4VyD1TJDk","_score":null,"_source":{"@timestamp":"2023-11-09T23:21:44.525384286Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:21:44.525+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1299","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572104525]},{"_index":"continuous-tests-pods-2023.11.09","_id":"glpktosBOvK4VyD1YL34","_score":null,"_source":{"@timestamp":"2023-11-09T23:21:47.526188331Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:21:47.525+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1300","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572107526]},{"_index":"continuous-tests-pods-2023.11.09","_id":"g1pktosBOvK4VyD1YL34","_score":null,"_source":{"@timestamp":"2023-11-09T23:21:50.532102243Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:21:50.531+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1301","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572110532]},{"_index":"continuous-tests-pods-2023.11.09","_id":"V1pktosBOvK4VyD1edQN","_score":null,"_source":{"@timestamp":"2023-11-09T23:21:53.533573011Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:21:53.533+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1302","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572113533]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WFpktosBOvK4VyD1edQN","_score":null,"_source":{"@timestamp":"2023-11-09T23:21:56.534642041Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:21:56.534+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1303","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572116534]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5VpktosBOvK4VyD1kekt","_score":null,"_source":{"@timestamp":"2023-11-09T23:21:59.535781213Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:21:59.535+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1304","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572119535]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5lpktosBOvK4VyD1kekt","_score":null,"_source":{"@timestamp":"2023-11-09T23:22:02.537535112Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:22:02.537+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1305","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572122537]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fltktosBOvK4VyD1qQtC","_score":null,"_source":{"@timestamp":"2023-11-09T23:22:05.539065056Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:22:05.538+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1306","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572125539]},{"_index":"continuous-tests-pods-2023.11.09","_id":"f1tktosBOvK4VyD1qQtC","_score":null,"_source":{"@timestamp":"2023-11-09T23:22:08.540518909Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:22:08.540+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1307","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572128540]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SFtktosBOvK4VyD1wUNW","_score":null,"_source":{"@timestamp":"2023-11-09T23:22:11.541697644Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:22:11.541+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1308","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572131541]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SVtktosBOvK4VyD1wUNW","_score":null,"_source":{"@timestamp":"2023-11-09T23:22:14.543191053Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:22:14.542+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1309","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572134543]},{"_index":"continuous-tests-pods-2023.11.09","_id":"D1tktosBOvK4VyD12Wts","_score":null,"_source":{"@timestamp":"2023-11-09T23:22:17.548917526Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:22:17.548+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1310","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572137548]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EFtktosBOvK4VyD12Wts","_score":null,"_source":{"@timestamp":"2023-11-09T23:22:20.549897372Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:22:20.549+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1311","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572140549]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EVtktosBOvK4VyD12Wts","_score":null,"_source":{"@timestamp":"2023-11-09T23:22:23.551339264Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:22:23.551+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1312","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572143551]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UltktosBOvK4VyD1-ZqQ","_score":null,"_source":{"@timestamp":"2023-11-09T23:22:26.552252592Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:22:26.551+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1313","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572146552]},{"_index":"continuous-tests-pods-2023.11.09","_id":"U1tktosBOvK4VyD1-ZqQ","_score":null,"_source":{"@timestamp":"2023-11-09T23:22:29.553171584Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:22:29.552+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1314","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572149553]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AVtltosBOvK4VyD1EfCo","_score":null,"_source":{"@timestamp":"2023-11-09T23:22:32.554987919Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:22:32.554+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1315","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572152554]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AltltosBOvK4VyD1EfCo","_score":null,"_source":{"@timestamp":"2023-11-09T23:22:35.555848576Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:22:35.555+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1316","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572155555]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HFxltosBOvK4VyD1KT_A","_score":null,"_source":{"@timestamp":"2023-11-09T23:22:38.556768310Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:22:38.556+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1317","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572158556]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HVxltosBOvK4VyD1KT_A","_score":null,"_source":{"@timestamp":"2023-11-09T23:22:41.558202108Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:22:41.557+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1318","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572161558]},{"_index":"continuous-tests-pods-2023.11.09","_id":"r1xltosBOvK4VyD1QYzc","_score":null,"_source":{"@timestamp":"2023-11-09T23:22:44.559453168Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:22:44.559+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1319","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572164559]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sFxltosBOvK4VyD1QYzc","_score":null,"_source":{"@timestamp":"2023-11-09T23:22:47.559929787Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:22:47.559+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1320","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572167559]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QFxltosBOvK4VyD1WeD0","_score":null,"_source":{"@timestamp":"2023-11-09T23:22:50.560910957Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:22:50.560+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1321","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572170560]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QVxltosBOvK4VyD1WeD0","_score":null,"_source":{"@timestamp":"2023-11-09T23:22:53.561915403Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:22:53.561+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1322","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572173561]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tFxltosBOvK4VyD1cv8Q","_score":null,"_source":{"@timestamp":"2023-11-09T23:22:56.563744068Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:22:56.563+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1323","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572176563]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tVxltosBOvK4VyD1cv8Q","_score":null,"_source":{"@timestamp":"2023-11-09T23:22:59.564974584Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:22:59.564+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1324","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572179564]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Cl1ltosBOvK4VyD1hhIm","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:02.566215730Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:02.565+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1325","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572182566]},{"_index":"continuous-tests-pods-2023.11.09","_id":"C11ltosBOvK4VyD1hhIm","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:03.855165634Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:03.854+00:00 Begin iterating blocks... topics=\"codex + discoveryengine\" tid=1 count=1326","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572183855]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DF1ltosBOvK4VyD1hhIm","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:03.855209663Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:03.855+00:00 Iterating blocks finished. topics=\"codex + discoveryengine\" tid=1 count=1327","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572183855]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DV1ltosBOvK4VyD1hhIm","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:03.855224480Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:03.855+00:00 About to sleep advertise loop topics=\"codex + discoveryengine\" tid=1 sleep=30m count=1328","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572183855]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Dl1ltosBOvK4VyD1hhIm","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:03.873677103Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:03.873+00:00 Cleaning up expired records tid=1 + count=1329","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572183873]},{"_index":"continuous-tests-pods-2023.11.09","_id":"D11ltosBOvK4VyD1hhIm","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:03.873718556Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:03.873+00:00 Cleaned up expired records tid=1 + size=0 count=1330","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572183873]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EF1ltosBOvK4VyD1hhIm","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:03.873724270Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:03.873+00:00 Cleaning up query iterator tid=1 + count=1331","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572183873]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EV1ltosBOvK4VyD1hhIm","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:03.873738154Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:03.873+00:00 Cleaning up orphaned records tid=1 + count=1332","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572183873]},{"_index":"continuous-tests-pods-2023.11.09","_id":"El1ltosBOvK4VyD1hhIm","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:03.873841115Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:03.873+00:00 Cleaning up query iterator tid=1 + count=1333","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572183873]},{"_index":"continuous-tests-pods-2023.11.09","_id":"E11ltosBOvK4VyD1hhIm","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:05.567810790Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:05.567+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1334","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572185567]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BV1ltosBOvK4VyD1nylH","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:08.568861743Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:08.568+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1335","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572188568]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Bl1ltosBOvK4VyD1nylH","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:11.570513784Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:11.570+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1336","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572191570]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4l1ltosBOvK4VyD1t0Bg","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:14.571488404Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:14.571+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1337","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572194571]},{"_index":"continuous-tests-pods-2023.11.09","_id":"411ltosBOvK4VyD1t0Bg","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:17.573270203Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:17.572+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1338","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572197573]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rV1ltosBOvK4VyD1z1p4","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:20.574791257Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:20.574+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1339","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572200574]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rl1ltosBOvK4VyD1z1p4","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:23.576282252Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:23.575+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1340","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572203576]},{"_index":"continuous-tests-pods-2023.11.09","_id":"r11ltosBOvK4VyD1z1p4","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:26.577453323Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:26.577+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1341","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572206577]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2F1ltosBOvK4VyD176qW","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:29.578434785Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:29.578+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1342","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572209578]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2V1ltosBOvK4VyD176qW","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:32.579500137Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:32.579+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1343","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572212579]},{"_index":"continuous-tests-pods-2023.11.09","_id":"n15mtosBOvK4VyD1Bwyw","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:35.581275758Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:35.580+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1344","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572215581]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oF5mtosBOvK4VyD1Bwyw","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:38.582097380Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:38.581+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1345","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572218582]},{"_index":"continuous-tests-pods-2023.11.09","_id":"315mtosBOvK4VyD1H0DG","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:41.582692282Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:41.582+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1346","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572221582]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4F5mtosBOvK4VyD1H0DG","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:44.583982698Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:44.583+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1347","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572224583]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2V5mtosBOvK4VyD1N1_b","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:47.584928368Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:47.584+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1348","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572227584]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2l5mtosBOvK4VyD1N1_b","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:50.586011592Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:50.585+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1349","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572230586]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Pl5mtosBOvK4VyD1T3T1","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:53.590981476Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:53.587+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1350","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572233590]},{"_index":"continuous-tests-pods-2023.11.09","_id":"P15mtosBOvK4VyD1T3T1","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:56.588779382Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:56.588+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1351","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572236588]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yl5mtosBOvK4VyD1aIsR","_score":null,"_source":{"@timestamp":"2023-11-09T23:23:59.590447522Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:23:59.590+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1352","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572239590]},{"_index":"continuous-tests-pods-2023.11.09","_id":"y15mtosBOvK4VyD1aIsR","_score":null,"_source":{"@timestamp":"2023-11-09T23:24:02.591537460Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:24:02.591+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1353","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572242591]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zF5mtosBOvK4VyD1aIsR","_score":null,"_source":{"@timestamp":"2023-11-09T23:24:05.593108738Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:24:05.592+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1354","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572245593]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5F5mtosBOvK4VyD1iMIx","_score":null,"_source":{"@timestamp":"2023-11-09T23:24:08.594097630Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:24:08.593+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1355","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572248594]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5V5mtosBOvK4VyD1iMIx","_score":null,"_source":{"@timestamp":"2023-11-09T23:24:11.595322460Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:24:11.594+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1356","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572251595]},{"_index":"continuous-tests-pods-2023.11.09","_id":"x15mtosBOvK4VyD1oPpE","_score":null,"_source":{"@timestamp":"2023-11-09T23:24:14.596310982Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:24:14.596+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1357","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572254596]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yF5mtosBOvK4VyD1oPpE","_score":null,"_source":{"@timestamp":"2023-11-09T23:24:17.597924514Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:24:17.597+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1358","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572257597]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ml9mtosBOvK4VyD1uBNe","_score":null,"_source":{"@timestamp":"2023-11-09T23:24:20.599081604Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:24:20.598+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1359","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572260599]},{"_index":"continuous-tests-pods-2023.11.09","_id":"M19mtosBOvK4VyD1uBNe","_score":null,"_source":{"@timestamp":"2023-11-09T23:24:23.605328130Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:24:23.605+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1360","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572263605]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cF9mtosBOvK4VyD10D53","_score":null,"_source":{"@timestamp":"2023-11-09T23:24:26.606462374Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:24:26.606+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1361","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572266606]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cV9mtosBOvK4VyD10D53","_score":null,"_source":{"@timestamp":"2023-11-09T23:24:29.607885035Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:24:29.607+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1362","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572269607]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xF9mtosBOvK4VyD16JCQ","_score":null,"_source":{"@timestamp":"2023-11-09T23:24:32.609520141Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:24:32.609+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1363","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572272609]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xV9mtosBOvK4VyD16JCQ","_score":null,"_source":{"@timestamp":"2023-11-09T23:24:35.610900923Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:24:35.610+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1364","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572275610]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YF9ntosBOvK4VyD1AOOp","_score":null,"_source":{"@timestamp":"2023-11-09T23:24:38.611920296Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:24:38.611+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1365","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572278611]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YV9ntosBOvK4VyD1AOOp","_score":null,"_source":{"@timestamp":"2023-11-09T23:24:41.613505447Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:24:41.613+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1366","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572281613]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-GBntosBOvK4VyD1FCfB","_score":null,"_source":{"@timestamp":"2023-11-09T23:24:44.614825479Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:24:44.614+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1367","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572284614]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-WBntosBOvK4VyD1FCfB","_score":null,"_source":{"@timestamp":"2023-11-09T23:24:47.616642301Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:24:47.616+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1368","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572287616]},{"_index":"continuous-tests-pods-2023.11.09","_id":"D2BntosBOvK4VyD1LHrZ","_score":null,"_source":{"@timestamp":"2023-11-09T23:24:50.617681555Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:24:50.617+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1369","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572290617]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EGBntosBOvK4VyD1LHrZ","_score":null,"_source":{"@timestamp":"2023-11-09T23:24:53.619324337Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:24:53.618+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1370","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572293619]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-GBntosBOvK4VyD1RJr3","_score":null,"_source":{"@timestamp":"2023-11-09T23:24:56.620503120Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:24:56.620+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1371","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572296620]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-WBntosBOvK4VyD1RJr3","_score":null,"_source":{"@timestamp":"2023-11-09T23:24:59.621811291Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:24:59.621+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1372","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572299621]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HmBntosBOvK4VyD1Xa8H","_score":null,"_source":{"@timestamp":"2023-11-09T23:25:02.622461684Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:25:02.621+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1373","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572302622]},{"_index":"continuous-tests-pods-2023.11.09","_id":"H2BntosBOvK4VyD1Xa8H","_score":null,"_source":{"@timestamp":"2023-11-09T23:25:05.623830405Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:25:05.623+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1374","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572305623]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NGBntosBOvK4VyD1dcce","_score":null,"_source":{"@timestamp":"2023-11-09T23:25:08.624564733Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:25:08.624+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1375","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572308624]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NWBntosBOvK4VyD1dcce","_score":null,"_source":{"@timestamp":"2023-11-09T23:25:11.630418033Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:25:11.630+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1376","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572311630]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8WBntosBOvK4VyD1jdw7","_score":null,"_source":{"@timestamp":"2023-11-09T23:25:14.632265066Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:25:14.631+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1377","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572314632]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8mBntosBOvK4VyD1jdw7","_score":null,"_source":{"@timestamp":"2023-11-09T23:25:17.633877438Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:25:17.633+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1378","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572317633]},{"_index":"continuous-tests-pods-2023.11.09","_id":"82BntosBOvK4VyD1jdw7","_score":null,"_source":{"@timestamp":"2023-11-09T23:25:20.634832172Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:25:20.634+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1379","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572320634]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VmFntosBOvK4VyD1rQxg","_score":null,"_source":{"@timestamp":"2023-11-09T23:25:23.635727484Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:25:23.635+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1380","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572323635]},{"_index":"continuous-tests-pods-2023.11.09","_id":"V2FntosBOvK4VyD1rQxg","_score":null,"_source":{"@timestamp":"2023-11-09T23:25:26.637454493Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:25:26.637+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1381","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572326637]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fWFntosBOvK4VyD1xU6E","_score":null,"_source":{"@timestamp":"2023-11-09T23:25:29.638864495Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:25:29.638+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1382","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572329638]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fmFntosBOvK4VyD1xU6E","_score":null,"_source":{"@timestamp":"2023-11-09T23:25:32.640515356Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:25:32.640+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1383","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572332640]},{"_index":"continuous-tests-pods-2023.11.09","_id":"b2FntosBOvK4VyD13ZeL","_score":null,"_source":{"@timestamp":"2023-11-09T23:25:35.641686705Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:25:35.641+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1384","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572335641]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cGFntosBOvK4VyD13ZeL","_score":null,"_source":{"@timestamp":"2023-11-09T23:25:38.642333047Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:25:38.641+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1385","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572338642]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ImFntosBOvK4VyD19cCg","_score":null,"_source":{"@timestamp":"2023-11-09T23:25:41.643242409Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:25:41.642+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1386","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572341643]},{"_index":"continuous-tests-pods-2023.11.09","_id":"I2FntosBOvK4VyD19cCg","_score":null,"_source":{"@timestamp":"2023-11-09T23:25:44.644675170Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:25:44.644+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1387","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572344644]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kmFotosBOvK4VyD1Deu2","_score":null,"_source":{"@timestamp":"2023-11-09T23:25:47.645657409Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:25:47.645+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1388","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572347645]},{"_index":"continuous-tests-pods-2023.11.09","_id":"k2FotosBOvK4VyD1Deu2","_score":null,"_source":{"@timestamp":"2023-11-09T23:25:50.647085302Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:25:50.646+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1389","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572350647]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xWJotosBOvK4VyD1JRLL","_score":null,"_source":{"@timestamp":"2023-11-09T23:25:53.648514607Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:25:53.648+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1390","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572353648]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xmJotosBOvK4VyD1JRLL","_score":null,"_source":{"@timestamp":"2023-11-09T23:25:56.649366984Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:25:56.649+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1391","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572356649]},{"_index":"continuous-tests-pods-2023.11.09","_id":"92JotosBOvK4VyD1OSPf","_score":null,"_source":{"@timestamp":"2023-11-09T23:25:59.655206423Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:25:59.654+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1392","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572359655]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-GJotosBOvK4VyD1OSPf","_score":null,"_source":{"@timestamp":"2023-11-09T23:26:02.656640623Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:26:02.656+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1393","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572362656]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oGJotosBOvK4VyD1UUL2","_score":null,"_source":{"@timestamp":"2023-11-09T23:26:05.657850566Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:26:05.657+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1394","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572365657]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oWJotosBOvK4VyD1UUL2","_score":null,"_source":{"@timestamp":"2023-11-09T23:26:08.659134596Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:26:08.658+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1395","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572368659]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UmJotosBOvK4VyD1anwO","_score":null,"_source":{"@timestamp":"2023-11-09T23:26:11.661044742Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:26:11.660+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1396","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572371661]},{"_index":"continuous-tests-pods-2023.11.09","_id":"U2JotosBOvK4VyD1anwO","_score":null,"_source":{"@timestamp":"2023-11-09T23:26:14.661993544Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:26:14.661+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1397","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572374661]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nmJotosBOvK4VyD1gqQo","_score":null,"_source":{"@timestamp":"2023-11-09T23:26:17.662966163Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:26:17.662+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1398","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572377662]},{"_index":"continuous-tests-pods-2023.11.09","_id":"n2JotosBOvK4VyD1gqQo","_score":null,"_source":{"@timestamp":"2023-11-09T23:26:20.664479928Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:26:20.664+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1399","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572380664]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GGJotosBOvK4VyD1mrtA","_score":null,"_source":{"@timestamp":"2023-11-09T23:26:23.665595296Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:26:23.665+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1400","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572383665]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GWJotosBOvK4VyD1mrtA","_score":null,"_source":{"@timestamp":"2023-11-09T23:26:26.666643334Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:26:26.666+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1401","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572386666]},{"_index":"continuous-tests-pods-2023.11.09","_id":"D2JotosBOvK4VyD1svZV","_score":null,"_source":{"@timestamp":"2023-11-09T23:26:29.668085433Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:26:29.667+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1402","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572389668]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EGJotosBOvK4VyD1svZV","_score":null,"_source":{"@timestamp":"2023-11-09T23:26:32.669232163Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:26:32.668+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1403","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572392669]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EWJotosBOvK4VyD1svZV","_score":null,"_source":{"@timestamp":"2023-11-09T23:26:35.670752053Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:26:35.670+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1404","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572395670]},{"_index":"continuous-tests-pods-2023.11.09","_id":"w2NotosBOvK4VyD10nN3","_score":null,"_source":{"@timestamp":"2023-11-09T23:26:38.671916968Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:26:38.671+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1405","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572398671]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xGNotosBOvK4VyD10nN3","_score":null,"_source":{"@timestamp":"2023-11-09T23:26:41.673559678Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:26:41.673+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1406","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572401673]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qWNotosBOvK4VyD16seN","_score":null,"_source":{"@timestamp":"2023-11-09T23:26:44.675002278Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:26:44.674+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1407","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572404675]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qmNotosBOvK4VyD16seO","_score":null,"_source":{"@timestamp":"2023-11-09T23:26:47.675998770Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:26:47.675+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1408","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572407675]},{"_index":"continuous-tests-pods-2023.11.09","_id":"K2RptosBOvK4VyD1Ahij","_score":null,"_source":{"@timestamp":"2023-11-09T23:26:50.677126098Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:26:50.676+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1409","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572410677]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LGRptosBOvK4VyD1Ahij","_score":null,"_source":{"@timestamp":"2023-11-09T23:26:53.682503684Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:26:53.682+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1410","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572413682]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VmRptosBOvK4VyD1Gji5","_score":null,"_source":{"@timestamp":"2023-11-09T23:26:56.684157266Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:26:56.683+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1411","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572416684]},{"_index":"continuous-tests-pods-2023.11.09","_id":"V2RptosBOvK4VyD1Gji5","_score":null,"_source":{"@timestamp":"2023-11-09T23:26:59.684648065Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:26:59.684+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1412","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572419684]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TmRptosBOvK4VyD1Mk7S","_score":null,"_source":{"@timestamp":"2023-11-09T23:27:02.686090571Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:27:02.685+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1413","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572422686]},{"_index":"continuous-tests-pods-2023.11.09","_id":"T2RptosBOvK4VyD1Mk7S","_score":null,"_source":{"@timestamp":"2023-11-09T23:27:05.687735743Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:27:05.687+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1414","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572425687]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RGRptosBOvK4VyD1SmXk","_score":null,"_source":{"@timestamp":"2023-11-09T23:27:08.688367434Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:27:08.687+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1415","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572428688]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RWRptosBOvK4VyD1SmXk","_score":null,"_source":{"@timestamp":"2023-11-09T23:27:11.689412634Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:27:11.689+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1416","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572431689]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zWRptosBOvK4VyD1Xnf8","_score":null,"_source":{"@timestamp":"2023-11-09T23:27:14.690790635Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:27:14.690+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1417","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572434690]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zmRptosBOvK4VyD1Xnf8","_score":null,"_source":{"@timestamp":"2023-11-09T23:27:17.691766693Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:27:17.691+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1418","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572437691]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-2RptosBOvK4VyD1d44X","_score":null,"_source":{"@timestamp":"2023-11-09T23:27:20.692656691Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:27:20.692+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1419","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572440692]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_GRptosBOvK4VyD1d44X","_score":null,"_source":{"@timestamp":"2023-11-09T23:27:23.694145477Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:27:23.693+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1420","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572443694]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-mRptosBOvK4VyD1j9Et","_score":null,"_source":{"@timestamp":"2023-11-09T23:27:26.699754137Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:27:26.699+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1421","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572446699]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-2RptosBOvK4VyD1j9Et","_score":null,"_source":{"@timestamp":"2023-11-09T23:27:29.701508128Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:27:29.701+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1422","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572449701]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iWVptosBOvK4VyD1pxNF","_score":null,"_source":{"@timestamp":"2023-11-09T23:27:32.702551239Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:27:32.702+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1423","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572452702]},{"_index":"continuous-tests-pods-2023.11.09","_id":"imVptosBOvK4VyD1pxNF","_score":null,"_source":{"@timestamp":"2023-11-09T23:27:35.703646026Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:27:35.703+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1424","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572455703]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RGVptosBOvK4VyD1v1xd","_score":null,"_source":{"@timestamp":"2023-11-09T23:27:38.704471245Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:27:38.704+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1425","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572458704]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RWVptosBOvK4VyD1v1xd","_score":null,"_source":{"@timestamp":"2023-11-09T23:27:41.706017161Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:27:41.705+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1426","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572461706]},{"_index":"continuous-tests-pods-2023.11.09","_id":"c2VptosBOvK4VyD11490","_score":null,"_source":{"@timestamp":"2023-11-09T23:27:44.707453201Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:27:44.707+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1427","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572464707]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dGVptosBOvK4VyD11490","_score":null,"_source":{"@timestamp":"2023-11-09T23:27:47.709491988Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:27:47.709+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1428","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572467709]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dWVptosBOvK4VyD11490","_score":null,"_source":{"@timestamp":"2023-11-09T23:27:50.710478077Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:27:50.710+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1429","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572470710]},{"_index":"continuous-tests-pods-2023.11.09","_id":"z2VptosBOvK4VyD1962a","_score":null,"_source":{"@timestamp":"2023-11-09T23:27:53.711407724Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:27:53.711+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1430","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572473711]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0GVptosBOvK4VyD1962a","_score":null,"_source":{"@timestamp":"2023-11-09T23:27:56.712952575Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:27:56.712+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1431","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572476712]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cWVqtosBOvK4VyD1D8Ox","_score":null,"_source":{"@timestamp":"2023-11-09T23:27:59.714309009Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:27:59.713+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1432","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572479714]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cmVqtosBOvK4VyD1D8Ox","_score":null,"_source":{"@timestamp":"2023-11-09T23:28:02.715499325Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:28:02.715+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1433","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572482715]},{"_index":"continuous-tests-pods-2023.11.09","_id":"c2VqtosBOvK4VyD1D8Ox","_score":null,"_source":{"@timestamp":"2023-11-09T23:28:03.875514468Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:28:03.874+00:00 Cleaning up expired records tid=1 + count=1434","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572483875]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dGVqtosBOvK4VyD1D8Ox","_score":null,"_source":{"@timestamp":"2023-11-09T23:28:03.875561119Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:28:03.875+00:00 Cleaned up expired records tid=1 + size=0 count=1435","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572483875]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dWVqtosBOvK4VyD1D8Ox","_score":null,"_source":{"@timestamp":"2023-11-09T23:28:03.875566551Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:28:03.875+00:00 Cleaning up query iterator tid=1 + count=1436","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572483875]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dmVqtosBOvK4VyD1D8Ox","_score":null,"_source":{"@timestamp":"2023-11-09T23:28:03.875571766Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:28:03.875+00:00 Cleaning up orphaned records tid=1 + count=1437","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572483875]},{"_index":"continuous-tests-pods-2023.11.09","_id":"d2VqtosBOvK4VyD1D8Ox","_score":null,"_source":{"@timestamp":"2023-11-09T23:28:03.875576783Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:28:03.875+00:00 Cleaning up query iterator tid=1 + count=1438","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572483875]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5mVqtosBOvK4VyD1I9rQ","_score":null,"_source":{"@timestamp":"2023-11-09T23:28:05.717066550Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:28:05.716+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1439","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572485717]},{"_index":"continuous-tests-pods-2023.11.09","_id":"52VqtosBOvK4VyD1I9rQ","_score":null,"_source":{"@timestamp":"2023-11-09T23:28:08.718517056Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:28:08.718+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1440","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572488718]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gWZqtosBOvK4VyD1Ow_q","_score":null,"_source":{"@timestamp":"2023-11-09T23:28:11.719986749Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:28:11.719+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1441","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572491719]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gmZqtosBOvK4VyD1Ow_q","_score":null,"_source":{"@timestamp":"2023-11-09T23:28:14.721527378Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:28:14.721+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1442","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572494721]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sGZqtosBOvK4VyD1VCwE","_score":null,"_source":{"@timestamp":"2023-11-09T23:28:17.722609409Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:28:17.722+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1443","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572497722]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sWZqtosBOvK4VyD1VCwE","_score":null,"_source":{"@timestamp":"2023-11-09T23:28:20.724074627Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:28:20.723+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1444","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572500724]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SmZqtosBOvK4VyD1bFUd","_score":null,"_source":{"@timestamp":"2023-11-09T23:28:23.725363675Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:28:23.725+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1445","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572503725]},{"_index":"continuous-tests-pods-2023.11.09","_id":"S2ZqtosBOvK4VyD1bFUd","_score":null,"_source":{"@timestamp":"2023-11-09T23:28:26.727165247Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:28:26.726+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1446","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572506727]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jGZqtosBOvK4VyD1hG8x","_score":null,"_source":{"@timestamp":"2023-11-09T23:28:29.728628467Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:28:29.728+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1447","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572509728]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jWZqtosBOvK4VyD1hG8x","_score":null,"_source":{"@timestamp":"2023-11-09T23:28:32.734160026Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:28:32.733+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1448","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572512734]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5mZqtosBOvK4VyD1nKtM","_score":null,"_source":{"@timestamp":"2023-11-09T23:28:35.735775102Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:28:35.735+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1449","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572515735]},{"_index":"continuous-tests-pods-2023.11.09","_id":"52ZqtosBOvK4VyD1nKtM","_score":null,"_source":{"@timestamp":"2023-11-09T23:28:38.737230620Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:28:38.736+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1450","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572518737]},{"_index":"continuous-tests-pods-2023.11.09","_id":"B2ZqtosBOvK4VyD1tP5n","_score":null,"_source":{"@timestamp":"2023-11-09T23:28:41.738342881Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:28:41.738+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1451","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572521738]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CGZqtosBOvK4VyD1tP5n","_score":null,"_source":{"@timestamp":"2023-11-09T23:28:44.739317281Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:28:44.738+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1452","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572524739]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5GdqtosBOvK4VyD1zE9-","_score":null,"_source":{"@timestamp":"2023-11-09T23:28:47.740647831Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:28:47.740+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1453","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572527740]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5WdqtosBOvK4VyD1zE9-","_score":null,"_source":{"@timestamp":"2023-11-09T23:28:50.742116586Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:28:50.741+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1454","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572530742]},{"_index":"continuous-tests-pods-2023.11.09","_id":"l2dqtosBOvK4VyD15KCW","_score":null,"_source":{"@timestamp":"2023-11-09T23:28:53.747447724Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:28:53.747+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1455","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572533747]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mGdqtosBOvK4VyD15KCW","_score":null,"_source":{"@timestamp":"2023-11-09T23:28:56.749020060Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:28:56.748+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1456","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572536749]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AGdqtosBOvK4VyD1_OOz","_score":null,"_source":{"@timestamp":"2023-11-09T23:28:59.750803742Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:28:59.750+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1457","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572539750]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AWdqtosBOvK4VyD1_OOz","_score":null,"_source":{"@timestamp":"2023-11-09T23:29:02.751578244Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:29:02.751+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1458","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572542751]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1GdrtosBOvK4VyD1EPbN","_score":null,"_source":{"@timestamp":"2023-11-09T23:29:05.752681271Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:29:05.752+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1459","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572545752]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1WdrtosBOvK4VyD1EPbO","_score":null,"_source":{"@timestamp":"2023-11-09T23:29:08.753814614Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:29:08.753+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1460","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572548753]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NGhrtosBOvK4VyD1KA7l","_score":null,"_source":{"@timestamp":"2023-11-09T23:29:11.755532727Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:29:11.755+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1461","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572551755]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NWhrtosBOvK4VyD1KA7l","_score":null,"_source":{"@timestamp":"2023-11-09T23:29:14.760860957Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:29:14.760+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1462","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572554760]},{"_index":"continuous-tests-pods-2023.11.09","_id":"42hrtosBOvK4VyD1QSMA","_score":null,"_source":{"@timestamp":"2023-11-09T23:29:17.762065393Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:29:17.761+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1463","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572557762]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5GhrtosBOvK4VyD1QSMA","_score":null,"_source":{"@timestamp":"2023-11-09T23:29:20.763287167Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:29:20.762+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1464","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572560763]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NGhrtosBOvK4VyD1WU8Z","_score":null,"_source":{"@timestamp":"2023-11-09T23:29:23.764364162Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:29:23.764+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1465","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572563764]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NWhrtosBOvK4VyD1WU8Z","_score":null,"_source":{"@timestamp":"2023-11-09T23:29:26.765914777Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:29:26.765+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1466","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572566765]},{"_index":"continuous-tests-pods-2023.11.09","_id":"X2hrtosBOvK4VyD1cY8z","_score":null,"_source":{"@timestamp":"2023-11-09T23:29:29.767412957Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:29:29.767+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1467","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572569767]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YGhrtosBOvK4VyD1cY8z","_score":null,"_source":{"@timestamp":"2023-11-09T23:29:32.768542340Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:29:32.768+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1468","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572572768]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DmhrtosBOvK4VyD1ieRL","_score":null,"_source":{"@timestamp":"2023-11-09T23:29:35.770230369Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:29:35.769+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1469","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572575770]},{"_index":"continuous-tests-pods-2023.11.09","_id":"D2hrtosBOvK4VyD1ieRL","_score":null,"_source":{"@timestamp":"2023-11-09T23:29:38.771759592Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:29:38.771+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1470","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572578771]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EGhrtosBOvK4VyD1ieRL","_score":null,"_source":{"@timestamp":"2023-11-09T23:29:41.772968837Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:29:41.772+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1471","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572581772]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JWlrtosBOvK4VyD1qSxs","_score":null,"_source":{"@timestamp":"2023-11-09T23:29:44.774181978Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:29:44.773+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1472","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572584774]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JmlrtosBOvK4VyD1qSxs","_score":null,"_source":{"@timestamp":"2023-11-09T23:29:47.775825485Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:29:47.775+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1473","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572587775]},{"_index":"continuous-tests-pods-2023.11.09","_id":"I2lrtosBOvK4VyD1wUOE","_score":null,"_source":{"@timestamp":"2023-11-09T23:29:50.777102895Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:29:50.776+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1474","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572590777]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JGlrtosBOvK4VyD1wUOE","_score":null,"_source":{"@timestamp":"2023-11-09T23:29:53.777879790Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:29:53.777+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1475","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572593777]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NWlrtosBOvK4VyD12Vmi","_score":null,"_source":{"@timestamp":"2023-11-09T23:29:56.779076315Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:29:56.778+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1476","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572596779]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NmlrtosBOvK4VyD12Vmi","_score":null,"_source":{"@timestamp":"2023-11-09T23:29:59.780165878Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:29:59.779+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1477","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572599780]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dmlrtosBOvK4VyD18XC2","_score":null,"_source":{"@timestamp":"2023-11-09T23:30:02.782007927Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:30:02.781+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1478","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572602782]},{"_index":"continuous-tests-pods-2023.11.09","_id":"d2lrtosBOvK4VyD18XC2","_score":null,"_source":{"@timestamp":"2023-11-09T23:30:05.782475975Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:30:05.782+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1479","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572605782]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QmlstosBOvK4VyD1CZjR","_score":null,"_source":{"@timestamp":"2023-11-09T23:30:08.788348677Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:30:08.788+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1480","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572608788]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Q2lstosBOvK4VyD1CZjR","_score":null,"_source":{"@timestamp":"2023-11-09T23:30:11.790149093Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:30:11.789+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1481","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572611790]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-WlstosBOvK4VyD1Ic_p","_score":null,"_source":{"@timestamp":"2023-11-09T23:30:14.791087366Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:30:14.790+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1482","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572614791]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-mlstosBOvK4VyD1Ic_p","_score":null,"_source":{"@timestamp":"2023-11-09T23:30:17.792927252Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:30:17.792+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1483","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572617792]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XmlstosBOvK4VyD1Nev_","_score":null,"_source":{"@timestamp":"2023-11-09T23:30:20.794367630Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:30:20.794+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1484","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572620794]},{"_index":"continuous-tests-pods-2023.11.09","_id":"X2lstosBOvK4VyD1Nev_","_score":null,"_source":{"@timestamp":"2023-11-09T23:30:23.795790655Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:30:23.795+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1485","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572623795]},{"_index":"continuous-tests-pods-2023.11.09","_id":"x2pstosBOvK4VyD1TgIX","_score":null,"_source":{"@timestamp":"2023-11-09T23:30:26.797784734Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:30:26.797+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1486","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572626797]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yGpstosBOvK4VyD1TgIX","_score":null,"_source":{"@timestamp":"2023-11-09T23:30:29.798747490Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:30:29.798+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1487","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572629798]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nWpstosBOvK4VyD1Zkg4","_score":null,"_source":{"@timestamp":"2023-11-09T23:30:32.800388492Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:30:32.800+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1488","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572632800]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nmpstosBOvK4VyD1Zkg4","_score":null,"_source":{"@timestamp":"2023-11-09T23:30:35.801518021Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:30:35.801+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1489","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572635801]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_WpstosBOvK4VyD1fp9N","_score":null,"_source":{"@timestamp":"2023-11-09T23:30:38.802091442Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:30:38.801+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1490","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572638802]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_mpstosBOvK4VyD1fp9N","_score":null,"_source":{"@timestamp":"2023-11-09T23:30:41.804070404Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:30:41.803+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1491","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572641804]},{"_index":"continuous-tests-pods-2023.11.09","_id":"R2pstosBOvK4VyD1lu9t","_score":null,"_source":{"@timestamp":"2023-11-09T23:30:44.805628260Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:30:44.805+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1492","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572644805]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SGpstosBOvK4VyD1lu9t","_score":null,"_source":{"@timestamp":"2023-11-09T23:30:47.807282330Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:30:47.806+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1493","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572647807]},{"_index":"continuous-tests-pods-2023.11.09","_id":"P2tstosBOvK4VyD1rj6E","_score":null,"_source":{"@timestamp":"2023-11-09T23:30:50.808482413Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:30:50.808+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1494","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572650808]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QGtstosBOvK4VyD1rj6E","_score":null,"_source":{"@timestamp":"2023-11-09T23:30:53.809700515Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:30:53.809+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1495","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572653809]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kGtstosBOvK4VyD1wnS5","_score":null,"_source":{"@timestamp":"2023-11-09T23:30:56.815903253Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:30:56.815+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1496","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572656815]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kWtstosBOvK4VyD1wnS5","_score":null,"_source":{"@timestamp":"2023-11-09T23:30:59.816837629Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:30:59.816+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1497","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572659816]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kWtstosBOvK4VyD12oq0","_score":null,"_source":{"@timestamp":"2023-11-09T23:31:02.818027686Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:31:02.817+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1498","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572662818]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kmtstosBOvK4VyD12oq0","_score":null,"_source":{"@timestamp":"2023-11-09T23:31:05.819171651Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:31:05.818+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1499","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572665819]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tmtstosBOvK4VyD18qHO","_score":null,"_source":{"@timestamp":"2023-11-09T23:31:08.820254421Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:31:08.819+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1500","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572668820]},{"_index":"continuous-tests-pods-2023.11.09","_id":"t2tstosBOvK4VyD18qHO","_score":null,"_source":{"@timestamp":"2023-11-09T23:31:11.821812264Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:31:11.821+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1501","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572671821]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pGtttosBOvK4VyD1Crfm","_score":null,"_source":{"@timestamp":"2023-11-09T23:31:14.823350166Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:31:14.822+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1502","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572674823]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pWtttosBOvK4VyD1Crfm","_score":null,"_source":{"@timestamp":"2023-11-09T23:31:17.824447994Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:31:17.824+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1503","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572677824]},{"_index":"continuous-tests-pods-2023.11.09","_id":"42tttosBOvK4VyD1ItH5","_score":null,"_source":{"@timestamp":"2023-11-09T23:31:20.825916046Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:31:20.825+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1504","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572680825]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5GtttosBOvK4VyD1ItH5","_score":null,"_source":{"@timestamp":"2023-11-09T23:31:23.826911935Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:31:23.826+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1505","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572683826]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Z2xttosBOvK4VyD1OxMR","_score":null,"_source":{"@timestamp":"2023-11-09T23:31:26.828124716Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:31:26.827+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1506","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572686828]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aGxttosBOvK4VyD1OxMR","_score":null,"_source":{"@timestamp":"2023-11-09T23:31:29.829832217Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:31:29.829+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1507","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572689829]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aWxttosBOvK4VyD1OxMR","_score":null,"_source":{"@timestamp":"2023-11-09T23:31:32.830823473Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:31:32.830+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1508","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572692830]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lmxttosBOvK4VyD1W3U1","_score":null,"_source":{"@timestamp":"2023-11-09T23:31:35.832277890Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:31:35.831+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1509","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572695832]},{"_index":"continuous-tests-pods-2023.11.09","_id":"l2xttosBOvK4VyD1W3U1","_score":null,"_source":{"@timestamp":"2023-11-09T23:31:38.832865066Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:31:38.832+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1510","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572698832]},{"_index":"continuous-tests-pods-2023.11.09","_id":"c2xttosBOvK4VyD1c6hQ","_score":null,"_source":{"@timestamp":"2023-11-09T23:31:41.834989970Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:31:41.834+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1511","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572701834]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dGxttosBOvK4VyD1c6hQ","_score":null,"_source":{"@timestamp":"2023-11-09T23:31:44.836350413Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:31:44.836+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1512","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572704836]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tWxttosBOvK4VyD1i9Vq","_score":null,"_source":{"@timestamp":"2023-11-09T23:31:47.838001002Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:31:47.837+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1513","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572707838]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tmxttosBOvK4VyD1i9Vq","_score":null,"_source":{"@timestamp":"2023-11-09T23:31:50.838724948Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:31:50.838+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1514","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572710838]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2GxttosBOvK4VyD1o-x_","_score":null,"_source":{"@timestamp":"2023-11-09T23:31:53.839920625Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:31:53.839+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1515","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572713839]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2WxttosBOvK4VyD1o-x_","_score":null,"_source":{"@timestamp":"2023-11-09T23:31:56.841505805Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:31:56.841+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1516","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572716841]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wG1ttosBOvK4VyD1uwKa","_score":null,"_source":{"@timestamp":"2023-11-09T23:31:59.847905342Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:31:59.847+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1517","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572719847]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wW1ttosBOvK4VyD1uwKa","_score":null,"_source":{"@timestamp":"2023-11-09T23:32:02.849137866Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:32:02.848+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1518","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572722849]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mm1ttosBOvK4VyD10xmy","_score":null,"_source":{"@timestamp":"2023-11-09T23:32:05.850083430Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:32:05.849+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1519","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572725850]},{"_index":"continuous-tests-pods-2023.11.09","_id":"m21ttosBOvK4VyD10xmy","_score":null,"_source":{"@timestamp":"2023-11-09T23:32:08.851591826Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:32:08.851+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1520","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572728851]},{"_index":"continuous-tests-pods-2023.11.09","_id":"M21ttosBOvK4VyD15zrK","_score":null,"_source":{"@timestamp":"2023-11-09T23:32:11.852702146Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:32:11.852+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1521","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572731852]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NG1ttosBOvK4VyD15zrK","_score":null,"_source":{"@timestamp":"2023-11-09T23:32:14.854209717Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:32:14.853+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1522","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572734854]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Y21ttosBOvK4VyD1_3Pi","_score":null,"_source":{"@timestamp":"2023-11-09T23:32:17.855710606Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:32:17.855+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1523","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572737855]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZG1ttosBOvK4VyD1_3Pi","_score":null,"_source":{"@timestamp":"2023-11-09T23:32:20.856712622Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:32:20.856+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1524","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572740856]},{"_index":"continuous-tests-pods-2023.11.09","_id":"v21utosBOvK4VyD1F5P5","_score":null,"_source":{"@timestamp":"2023-11-09T23:32:23.858368007Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:32:23.858+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1525","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572743858]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wG1utosBOvK4VyD1F5P5","_score":null,"_source":{"@timestamp":"2023-11-09T23:32:26.859626742Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:32:26.859+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1526","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572746859]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Km1utosBOvK4VyD1MKwQ","_score":null,"_source":{"@timestamp":"2023-11-09T23:32:29.861376727Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:32:29.860+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1527","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572749861]},{"_index":"continuous-tests-pods-2023.11.09","_id":"K21utosBOvK4VyD1MKwQ","_score":null,"_source":{"@timestamp":"2023-11-09T23:32:32.862793126Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:32:32.862+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1528","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572752862]},{"_index":"continuous-tests-pods-2023.11.09","_id":"X25utosBOvK4VyD1SAQn","_score":null,"_source":{"@timestamp":"2023-11-09T23:32:35.863610159Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:32:35.863+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1529","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572755863]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YG5utosBOvK4VyD1SAQo","_score":null,"_source":{"@timestamp":"2023-11-09T23:32:38.864798278Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:32:38.864+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1530","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572758864]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_G5utosBOvK4VyD1YFxB","_score":null,"_source":{"@timestamp":"2023-11-09T23:32:41.866699744Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:32:41.866+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1531","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572761866]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_W5utosBOvK4VyD1YFxB","_score":null,"_source":{"@timestamp":"2023-11-09T23:32:44.868368870Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:32:44.868+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1532","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572764868]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_m5utosBOvK4VyD1YFxB","_score":null,"_source":{"@timestamp":"2023-11-09T23:32:47.870080204Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:32:47.869+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1533","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572767870]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZW5utosBOvK4VyD1gMZl","_score":null,"_source":{"@timestamp":"2023-11-09T23:32:50.871679078Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:32:50.871+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1534","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572770871]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Zm5utosBOvK4VyD1gMZl","_score":null,"_source":{"@timestamp":"2023-11-09T23:32:53.872812923Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:32:53.872+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1535","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572773872]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eG9utosBOvK4VyD1mBJ-","_score":null,"_source":{"@timestamp":"2023-11-09T23:32:56.874732665Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:32:56.874+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1536","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572776874]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eW9utosBOvK4VyD1mBJ-","_score":null,"_source":{"@timestamp":"2023-11-09T23:32:59.876687275Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:32:59.876+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1537","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572779876]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mW9utosBOvK4VyD1sCqV","_score":null,"_source":{"@timestamp":"2023-11-09T23:33:02.878299977Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:33:02.877+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1538","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572782878]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mm9utosBOvK4VyD1sCqV","_score":null,"_source":{"@timestamp":"2023-11-09T23:33:03.876459482Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:33:03.876+00:00 Cleaning up expired records tid=1 + count=1539","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572783876]},{"_index":"continuous-tests-pods-2023.11.09","_id":"m29utosBOvK4VyD1sCqV","_score":null,"_source":{"@timestamp":"2023-11-09T23:33:03.876615836Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:33:03.876+00:00 Cleaned up expired records tid=1 + size=0 count=1540","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572783876]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nG9utosBOvK4VyD1sCqV","_score":null,"_source":{"@timestamp":"2023-11-09T23:33:03.876643945Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:33:03.876+00:00 Cleaning up query iterator tid=1 + count=1541","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572783876]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nW9utosBOvK4VyD1sCqV","_score":null,"_source":{"@timestamp":"2023-11-09T23:33:03.876654485Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:33:03.876+00:00 Cleaning up orphaned records tid=1 + count=1542","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572783876]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nm9utosBOvK4VyD1sCqV","_score":null,"_source":{"@timestamp":"2023-11-09T23:33:03.876828540Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:33:03.876+00:00 Cleaning up query iterator tid=1 + count=1543","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572783876]},{"_index":"continuous-tests-pods-2023.11.09","_id":"n29utosBOvK4VyD1sCqV","_score":null,"_source":{"@timestamp":"2023-11-09T23:33:05.879014764Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:33:05.878+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1544","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572785879]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sW9utosBOvK4VyD1yECr","_score":null,"_source":{"@timestamp":"2023-11-09T23:33:08.880814373Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:33:08.880+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1545","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572788880]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sm9utosBOvK4VyD1yECr","_score":null,"_source":{"@timestamp":"2023-11-09T23:33:11.882860502Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:33:11.882+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1546","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572791882]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vW9utosBOvK4VyD14FfG","_score":null,"_source":{"@timestamp":"2023-11-09T23:33:14.884360102Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:33:14.884+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1547","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572794884]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vm9utosBOvK4VyD14FfG","_score":null,"_source":{"@timestamp":"2023-11-09T23:33:17.886017960Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:33:17.885+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1548","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572797886]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8W9utosBOvK4VyD1-G_i","_score":null,"_source":{"@timestamp":"2023-11-09T23:33:20.887405059Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:33:20.887+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1549","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572800887]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8m9utosBOvK4VyD1-G_i","_score":null,"_source":{"@timestamp":"2023-11-09T23:33:23.888628170Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:33:23.888+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1550","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572803888]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ym9vtosBOvK4VyD1DKT6","_score":null,"_source":{"@timestamp":"2023-11-09T23:33:26.890250094Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:33:26.889+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1551","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572806890]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Y29vtosBOvK4VyD1DKT6","_score":null,"_source":{"@timestamp":"2023-11-09T23:33:29.891209601Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:33:29.890+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1552","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572809891]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5m9vtosBOvK4VyD1JeoT","_score":null,"_source":{"@timestamp":"2023-11-09T23:33:32.892488817Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:33:32.892+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1553","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572812892]},{"_index":"continuous-tests-pods-2023.11.09","_id":"529vtosBOvK4VyD1JeoT","_score":null,"_source":{"@timestamp":"2023-11-09T23:33:35.894226314Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:33:35.893+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1554","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572815894]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UXBvtosBOvK4VyD1PTIn","_score":null,"_source":{"@timestamp":"2023-11-09T23:33:38.895253587Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:33:38.894+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1555","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572818895]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UnBvtosBOvK4VyD1PTIn","_score":null,"_source":{"@timestamp":"2023-11-09T23:33:41.897353799Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:33:41.896+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1556","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572821897]},{"_index":"continuous-tests-pods-2023.11.09","_id":"p3BvtosBOvK4VyD1VWlD","_score":null,"_source":{"@timestamp":"2023-11-09T23:33:44.898647396Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:33:44.898+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1557","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572824898]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qHBvtosBOvK4VyD1VWlD","_score":null,"_source":{"@timestamp":"2023-11-09T23:33:47.900567698Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:33:47.900+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1558","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572827900]},{"_index":"continuous-tests-pods-2023.11.09","_id":"snBvtosBOvK4VyD1bYBi","_score":null,"_source":{"@timestamp":"2023-11-09T23:33:50.902457711Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:33:50.902+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1559","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572830902]},{"_index":"continuous-tests-pods-2023.11.09","_id":"s3BvtosBOvK4VyD1bYBi","_score":null,"_source":{"@timestamp":"2023-11-09T23:33:53.903423757Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:33:53.903+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1560","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572833903]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WXBvtosBOvK4VyD1hZZ8","_score":null,"_source":{"@timestamp":"2023-11-09T23:33:56.904872695Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:33:56.904+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1561","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572836904]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WnBvtosBOvK4VyD1hZZ8","_score":null,"_source":{"@timestamp":"2023-11-09T23:33:59.907017497Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:33:59.906+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1562","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572839907]},{"_index":"continuous-tests-pods-2023.11.09","_id":"R3BvtosBOvK4VyD1maqU","_score":null,"_source":{"@timestamp":"2023-11-09T23:34:02.907251686Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:34:02.906+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1563","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572842907]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SHBvtosBOvK4VyD1maqU","_score":null,"_source":{"@timestamp":"2023-11-09T23:34:05.908374559Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:34:05.908+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1564","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572845908]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0HBvtosBOvK4VyD1scqs","_score":null,"_source":{"@timestamp":"2023-11-09T23:34:08.909938013Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:34:08.909+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1565","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572848909]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0XBvtosBOvK4VyD1scqs","_score":null,"_source":{"@timestamp":"2023-11-09T23:34:11.911079723Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:34:11.910+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1566","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572851911]},{"_index":"continuous-tests-pods-2023.11.09","_id":"w3FvtosBOvK4VyD1yQLF","_score":null,"_source":{"@timestamp":"2023-11-09T23:34:14.912047850Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:34:14.911+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1567","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572854912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xHFvtosBOvK4VyD1yQLF","_score":null,"_source":{"@timestamp":"2023-11-09T23:34:17.913492785Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:34:17.913+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1568","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572857913]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NXFvtosBOvK4VyD14Snc","_score":null,"_source":{"@timestamp":"2023-11-09T23:34:20.915144479Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:34:20.914+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1569","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572860915]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NnFvtosBOvK4VyD14Snc","_score":null,"_source":{"@timestamp":"2023-11-09T23:34:23.916574900Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:34:23.916+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1570","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572863916]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7nFvtosBOvK4VyD1-UH3","_score":null,"_source":{"@timestamp":"2023-11-09T23:34:26.918402127Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:34:26.918+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1571","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572866918]},{"_index":"continuous-tests-pods-2023.11.09","_id":"73FvtosBOvK4VyD1-UH3","_score":null,"_source":{"@timestamp":"2023-11-09T23:34:29.919737983Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:34:29.919+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1572","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572869919]},{"_index":"continuous-tests-pods-2023.11.09","_id":"znFwtosBOvK4VyD1En8Q","_score":null,"_source":{"@timestamp":"2023-11-09T23:34:32.920876122Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:34:32.920+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1573","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572872920]},{"_index":"continuous-tests-pods-2023.11.09","_id":"z3FwtosBOvK4VyD1En8Q","_score":null,"_source":{"@timestamp":"2023-11-09T23:34:35.923072642Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:34:35.922+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1574","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572875923]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0HFwtosBOvK4VyD1En8Q","_score":null,"_source":{"@timestamp":"2023-11-09T23:34:38.924406713Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:34:38.924+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1575","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572878924]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jHJwtosBOvK4VyD1MgI2","_score":null,"_source":{"@timestamp":"2023-11-09T23:34:41.929645777Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:34:41.929+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1576","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572881929]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jXJwtosBOvK4VyD1MgI2","_score":null,"_source":{"@timestamp":"2023-11-09T23:34:44.930801221Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:34:44.930+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1577","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572884930]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wHJwtosBOvK4VyD1SldL","_score":null,"_source":{"@timestamp":"2023-11-09T23:34:47.932005562Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:34:47.931+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1578","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572887932]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wXJwtosBOvK4VyD1SldL","_score":null,"_source":{"@timestamp":"2023-11-09T23:34:50.933598062Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:34:50.933+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1579","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572890933]},{"_index":"continuous-tests-pods-2023.11.09","_id":"B3JwtosBOvK4VyD1Yqdk","_score":null,"_source":{"@timestamp":"2023-11-09T23:34:53.935377815Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:34:53.935+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1580","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572893935]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CHJwtosBOvK4VyD1Yqdk","_score":null,"_source":{"@timestamp":"2023-11-09T23:34:56.937359291Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:34:56.936+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1581","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572896937]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RXJwtosBOvK4VyD1er97","_score":null,"_source":{"@timestamp":"2023-11-09T23:34:59.938616181Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:34:59.938+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1582","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572899938]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RnJwtosBOvK4VyD1er97","_score":null,"_source":{"@timestamp":"2023-11-09T23:35:02.945921472Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:35:02.945+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1583","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572902945]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7HJwtosBOvK4VyD1ktWT","_score":null,"_source":{"@timestamp":"2023-11-09T23:35:05.947659293Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:35:05.947+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1584","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572905947]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7XJwtosBOvK4VyD1ktWT","_score":null,"_source":{"@timestamp":"2023-11-09T23:35:08.949517354Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:35:08.949+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1585","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572908949]},{"_index":"continuous-tests-pods-2023.11.09","_id":"53JwtosBOvK4VyD1quyr","_score":null,"_source":{"@timestamp":"2023-11-09T23:35:11.951328161Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:35:11.951+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1586","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572911951]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6HJwtosBOvK4VyD1quys","_score":null,"_source":{"@timestamp":"2023-11-09T23:35:14.953311130Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:35:14.952+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1587","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572914953]},{"_index":"continuous-tests-pods-2023.11.09","_id":"33JwtosBOvK4VyD1vv7D","_score":null,"_source":{"@timestamp":"2023-11-09T23:35:17.954726286Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:35:17.954+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1588","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572917954]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4HJwtosBOvK4VyD1vv7D","_score":null,"_source":{"@timestamp":"2023-11-09T23:35:20.956365279Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:35:20.956+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1589","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572920956]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yXNwtosBOvK4VyD11iHa","_score":null,"_source":{"@timestamp":"2023-11-09T23:35:23.957095314Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:35:23.956+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1590","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572923957]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ynNwtosBOvK4VyD11iHa","_score":null,"_source":{"@timestamp":"2023-11-09T23:35:26.959037501Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:35:26.958+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1591","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572926959]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Q3NwtosBOvK4VyD17mTx","_score":null,"_source":{"@timestamp":"2023-11-09T23:35:29.965353051Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:35:29.965+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1592","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572929965]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RHNwtosBOvK4VyD17mTx","_score":null,"_source":{"@timestamp":"2023-11-09T23:35:32.966673117Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:35:32.966+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1593","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572932966]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OnNxtosBOvK4VyD1B7oG","_score":null,"_source":{"@timestamp":"2023-11-09T23:35:35.968460526Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:35:35.968+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1594","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572935968]},{"_index":"continuous-tests-pods-2023.11.09","_id":"O3NxtosBOvK4VyD1B7oG","_score":null,"_source":{"@timestamp":"2023-11-09T23:35:38.970493321Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:35:38.970+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1595","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572938970]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KXNxtosBOvK4VyD1H-Qg","_score":null,"_source":{"@timestamp":"2023-11-09T23:35:41.972169247Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:35:41.971+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1596","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572941972]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KnNxtosBOvK4VyD1H-Qg","_score":null,"_source":{"@timestamp":"2023-11-09T23:35:44.974299811Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:35:44.974+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1597","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572944974]},{"_index":"continuous-tests-pods-2023.11.09","_id":"unRxtosBOvK4VyD1NxU5","_score":null,"_source":{"@timestamp":"2023-11-09T23:35:47.976626111Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:35:47.976+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1598","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572947976]},{"_index":"continuous-tests-pods-2023.11.09","_id":"u3RxtosBOvK4VyD1NxU5","_score":null,"_source":{"@timestamp":"2023-11-09T23:35:50.982700996Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:35:50.982+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1599","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572950982]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vHRxtosBOvK4VyD1NxU5","_score":null,"_source":{"@timestamp":"2023-11-09T23:35:53.984973411Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:35:53.984+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1600","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572953984]},{"_index":"continuous-tests-pods-2023.11.09","_id":"C3RxtosBOvK4VyD1VzNc","_score":null,"_source":{"@timestamp":"2023-11-09T23:35:56.986321811Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:35:56.986+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1601","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572956986]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DHRxtosBOvK4VyD1VzNc","_score":null,"_source":{"@timestamp":"2023-11-09T23:35:59.988042844Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:35:59.987+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1602","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572959988]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mHRxtosBOvK4VyD1b0hz","_score":null,"_source":{"@timestamp":"2023-11-09T23:36:02.989846707Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:36:02.989+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1603","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572962989]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mXRxtosBOvK4VyD1b0hz","_score":null,"_source":{"@timestamp":"2023-11-09T23:36:05.991591208Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:36:05.991+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1604","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572965991]},{"_index":"continuous-tests-pods-2023.11.09","_id":"I3RxtosBOvK4VyD1h2yK","_score":null,"_source":{"@timestamp":"2023-11-09T23:36:08.993827706Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:36:08.993+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1605","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572968993]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JHRxtosBOvK4VyD1h2yK","_score":null,"_source":{"@timestamp":"2023-11-09T23:36:11.995165895Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:36:11.994+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1606","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572971995]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZHRxtosBOvK4VyD1n6Sk","_score":null,"_source":{"@timestamp":"2023-11-09T23:36:14.996967811Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:36:14.996+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1607","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572974996]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZXRxtosBOvK4VyD1n6Sk","_score":null,"_source":{"@timestamp":"2023-11-09T23:36:17.997182830Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:36:17.996+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1608","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572977997]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hXRxtosBOvK4VyD1t8m8","_score":null,"_source":{"@timestamp":"2023-11-09T23:36:20.997425940Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:36:20.997+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1609","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572980997]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hnRxtosBOvK4VyD1t8m8","_score":null,"_source":{"@timestamp":"2023-11-09T23:36:23.997701402Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:36:23.997+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1610","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572983997]},{"_index":"continuous-tests-pods-2023.11.09","_id":"O3RxtosBOvK4VyD1z-DU","_score":null,"_source":{"@timestamp":"2023-11-09T23:36:26.998201060Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:36:26.997+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1611","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572986998]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PHRxtosBOvK4VyD1z-DU","_score":null,"_source":{"@timestamp":"2023-11-09T23:36:29.998907829Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:36:29.998+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1612","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572989998]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pnVxtosBOvK4VyD15CQl","_score":null,"_source":{"@timestamp":"2023-11-09T23:36:32.999654651Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:36:32.999+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1613","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572992999]},{"_index":"continuous-tests-pods-2023.11.09","_id":"p3VxtosBOvK4VyD15CQl","_score":null,"_source":{"@timestamp":"2023-11-09T23:36:36.000820239Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:36:36.000+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1614","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572996000]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iHVxtosBOvK4VyD1_HU5","_score":null,"_source":{"@timestamp":"2023-11-09T23:36:39.002076983Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:36:39.001+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1615","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699572999002]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iXVxtosBOvK4VyD1_HU5","_score":null,"_source":{"@timestamp":"2023-11-09T23:36:42.003031266Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:36:42.002+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1616","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573002003]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zHVytosBOvK4VyD1FMlQ","_score":null,"_source":{"@timestamp":"2023-11-09T23:36:45.003784102Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:36:45.003+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1617","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573005003]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zXVytosBOvK4VyD1FMlQ","_score":null,"_source":{"@timestamp":"2023-11-09T23:36:48.009224719Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:36:48.008+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1618","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573008009]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xXZytosBOvK4VyD1LBto","_score":null,"_source":{"@timestamp":"2023-11-09T23:36:51.010061919Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:36:51.009+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1619","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573011010]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xnZytosBOvK4VyD1LBto","_score":null,"_source":{"@timestamp":"2023-11-09T23:36:54.010920054Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:36:54.010+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1620","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573014010]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EHZytosBOvK4VyD1RFJ9","_score":null,"_source":{"@timestamp":"2023-11-09T23:36:57.011279406Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:36:57.010+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1621","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573017011]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EXZytosBOvK4VyD1RFJ9","_score":null,"_source":{"@timestamp":"2023-11-09T23:37:00.011905144Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:37:00.011+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1622","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573020011]},{"_index":"continuous-tests-pods-2023.11.09","_id":"o3ZytosBOvK4VyD1XGuW","_score":null,"_source":{"@timestamp":"2023-11-09T23:37:03.012432885Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:37:03.011+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1623","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573023012]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pHZytosBOvK4VyD1XGuW","_score":null,"_source":{"@timestamp":"2023-11-09T23:37:06.013119819Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:37:06.012+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1624","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573026013]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GXZytosBOvK4VyD1cIGv","_score":null,"_source":{"@timestamp":"2023-11-09T23:37:09.015461305Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:37:09.013+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1625","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573029015]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GnZytosBOvK4VyD1cIGv","_score":null,"_source":{"@timestamp":"2023-11-09T23:37:12.015061343Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:37:12.014+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1626","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573032015]},{"_index":"continuous-tests-pods-2023.11.09","_id":"U3ZytosBOvK4VyD1iJnI","_score":null,"_source":{"@timestamp":"2023-11-09T23:37:15.020606561Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:37:15.020+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1627","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573035020]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VHZytosBOvK4VyD1iJnI","_score":null,"_source":{"@timestamp":"2023-11-09T23:37:18.021068679Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:37:18.020+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1628","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573038021]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qnZytosBOvK4VyD1oLHf","_score":null,"_source":{"@timestamp":"2023-11-09T23:37:21.021379611Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:37:21.021+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1629","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573041021]},{"_index":"continuous-tests-pods-2023.11.09","_id":"q3ZytosBOvK4VyD1oLHf","_score":null,"_source":{"@timestamp":"2023-11-09T23:37:24.021858268Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:37:24.021+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1630","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573044021]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Y3ZytosBOvK4VyD1uOn0","_score":null,"_source":{"@timestamp":"2023-11-09T23:37:27.022295048Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:37:27.021+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1631","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573047022]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZHZytosBOvK4VyD1uOn0","_score":null,"_source":{"@timestamp":"2023-11-09T23:37:30.022963209Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:37:30.022+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1632","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573050022]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kndytosBOvK4VyD10SwO","_score":null,"_source":{"@timestamp":"2023-11-09T23:37:33.023651759Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:37:33.023+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1633","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573053023]},{"_index":"continuous-tests-pods-2023.11.09","_id":"k3dytosBOvK4VyD10SwO","_score":null,"_source":{"@timestamp":"2023-11-09T23:37:36.028570006Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:37:36.028+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1634","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573056028]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vXdytosBOvK4VyD16Xck","_score":null,"_source":{"@timestamp":"2023-11-09T23:37:39.029601137Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:37:39.029+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1635","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573059029]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vndytosBOvK4VyD16Xck","_score":null,"_source":{"@timestamp":"2023-11-09T23:37:42.030692856Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:37:42.030+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1636","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573062030]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9HdytosBOvK4VyD1_Zs8","_score":null,"_source":{"@timestamp":"2023-11-09T23:37:45.031870362Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:37:45.031+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1637","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573065031]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9XdytosBOvK4VyD1_Zs8","_score":null,"_source":{"@timestamp":"2023-11-09T23:37:48.032118535Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:37:48.031+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1638","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573068032]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vXdztosBOvK4VyD1FdBS","_score":null,"_source":{"@timestamp":"2023-11-09T23:37:51.033019891Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:37:51.032+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1639","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573071033]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vndztosBOvK4VyD1FdBS","_score":null,"_source":{"@timestamp":"2023-11-09T23:37:54.033945078Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:37:54.033+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1640","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573074033]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zndztosBOvK4VyD1Ledo","_score":null,"_source":{"@timestamp":"2023-11-09T23:37:57.034760483Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:37:57.034+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1641","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573077034]},{"_index":"continuous-tests-pods-2023.11.09","_id":"z3dztosBOvK4VyD1Ledo","_score":null,"_source":{"@timestamp":"2023-11-09T23:38:00.035056536Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:38:00.034+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1642","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573080035]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JnhztosBOvK4VyD1RQCA","_score":null,"_source":{"@timestamp":"2023-11-09T23:38:03.035973549Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:38:03.035+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1643","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573083035]},{"_index":"continuous-tests-pods-2023.11.09","_id":"J3hztosBOvK4VyD1RQCA","_score":null,"_source":{"@timestamp":"2023-11-09T23:38:03.877175011Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:38:03.876+00:00 Cleaning up expired records tid=1 + count=1644","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573083877]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KHhztosBOvK4VyD1RQCA","_score":null,"_source":{"@timestamp":"2023-11-09T23:38:03.877374843Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:38:03.877+00:00 Cleaned up expired records tid=1 + size=0 count=1645","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573083877]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KXhztosBOvK4VyD1RQCA","_score":null,"_source":{"@timestamp":"2023-11-09T23:38:03.877387109Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:38:03.877+00:00 Cleaning up query iterator tid=1 + count=1646","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573083877]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KnhztosBOvK4VyD1RQCA","_score":null,"_source":{"@timestamp":"2023-11-09T23:38:03.877400084Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:38:03.877+00:00 Cleaning up orphaned records tid=1 + count=1647","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573083877]},{"_index":"continuous-tests-pods-2023.11.09","_id":"K3hztosBOvK4VyD1RQCA","_score":null,"_source":{"@timestamp":"2023-11-09T23:38:03.877497376Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:38:03.877+00:00 Cleaning up query iterator tid=1 + count=1648","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573083877]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LHhztosBOvK4VyD1RQCA","_score":null,"_source":{"@timestamp":"2023-11-09T23:38:06.036616109Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:38:06.036+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1649","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573086036]},{"_index":"continuous-tests-pods-2023.11.09","_id":"E3hztosBOvK4VyD1XSWW","_score":null,"_source":{"@timestamp":"2023-11-09T23:38:09.037520711Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:38:09.037+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1650","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573089037]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FHhztosBOvK4VyD1XSWW","_score":null,"_source":{"@timestamp":"2023-11-09T23:38:12.038533185Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:38:12.038+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1651","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573092038]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_3hztosBOvK4VyD1dVyx","_score":null,"_source":{"@timestamp":"2023-11-09T23:38:15.039618580Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:38:15.039+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1652","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573095039]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AHhztosBOvK4VyD1dV2x","_score":null,"_source":{"@timestamp":"2023-11-09T23:38:18.039933242Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:38:18.039+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1653","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573098039]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AXhztosBOvK4VyD1dV2x","_score":null,"_source":{"@timestamp":"2023-11-09T23:38:21.041082790Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:38:21.040+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1654","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573101041]},{"_index":"continuous-tests-pods-2023.11.09","_id":"onhztosBOvK4VyD1lY_V","_score":null,"_source":{"@timestamp":"2023-11-09T23:38:24.041678880Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:38:24.041+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1655","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573104041]},{"_index":"continuous-tests-pods-2023.11.09","_id":"o3hztosBOvK4VyD1lY_V","_score":null,"_source":{"@timestamp":"2023-11-09T23:38:27.043072591Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:38:27.042+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1656","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573107043]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fXhztosBOvK4VyD1rbfx","_score":null,"_source":{"@timestamp":"2023-11-09T23:38:30.043449088Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:38:30.043+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1657","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573110043]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fnhztosBOvK4VyD1rbfx","_score":null,"_source":{"@timestamp":"2023-11-09T23:38:33.044207680Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:38:33.043+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1658","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573113044]},{"_index":"continuous-tests-pods-2023.11.09","_id":"43lztosBOvK4VyD1xg8M","_score":null,"_source":{"@timestamp":"2023-11-09T23:38:36.049043084Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:38:36.048+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1659","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573116049]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5HlztosBOvK4VyD1xg8N","_score":null,"_source":{"@timestamp":"2023-11-09T23:38:39.049573027Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:38:39.049+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1660","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573119049]},{"_index":"continuous-tests-pods-2023.11.09","_id":"83lztosBOvK4VyD13mci","_score":null,"_source":{"@timestamp":"2023-11-09T23:38:42.050489441Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:38:42.050+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1661","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573122050]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9HlztosBOvK4VyD13mci","_score":null,"_source":{"@timestamp":"2023-11-09T23:38:45.051649159Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:38:45.051+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1662","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573125051]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CnlztosBOvK4VyD19r83","_score":null,"_source":{"@timestamp":"2023-11-09T23:38:48.052106761Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:38:48.051+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1663","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573128052]},{"_index":"continuous-tests-pods-2023.11.09","_id":"C3lztosBOvK4VyD19r83","_score":null,"_source":{"@timestamp":"2023-11-09T23:38:51.053028626Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:38:51.052+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1664","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573131053]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hHp0tosBOvK4VyD1Dg9P","_score":null,"_source":{"@timestamp":"2023-11-09T23:38:54.053428272Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:38:54.053+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1665","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573134053]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hXp0tosBOvK4VyD1Dg9P","_score":null,"_source":{"@timestamp":"2023-11-09T23:38:57.053844785Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:38:57.053+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1666","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573137053]},{"_index":"continuous-tests-pods-2023.11.09","_id":"83p0tosBOvK4VyD1IiVl","_score":null,"_source":{"@timestamp":"2023-11-09T23:39:00.054311702Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:39:00.053+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1667","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573140054]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9Hp0tosBOvK4VyD1IiVl","_score":null,"_source":{"@timestamp":"2023-11-09T23:39:03.060348518Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:39:03.059+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1668","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573143060]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rXp0tosBOvK4VyD1Oj59","_score":null,"_source":{"@timestamp":"2023-11-09T23:39:06.061337304Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:39:06.060+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1669","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573146061]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rnp0tosBOvK4VyD1Oj59","_score":null,"_source":{"@timestamp":"2023-11-09T23:39:09.062168587Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:39:09.061+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1670","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573149062]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8np0tosBOvK4VyD1UlaW","_score":null,"_source":{"@timestamp":"2023-11-09T23:39:12.062990900Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:39:12.062+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1671","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573152062]},{"_index":"continuous-tests-pods-2023.11.09","_id":"83p0tosBOvK4VyD1UlaW","_score":null,"_source":{"@timestamp":"2023-11-09T23:39:15.063646350Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:39:15.063+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1672","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573155063]},{"_index":"continuous-tests-pods-2023.11.09","_id":"i3p0tosBOvK4VyD1am-x","_score":null,"_source":{"@timestamp":"2023-11-09T23:39:18.064699066Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:39:18.064+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1673","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573158064]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jHp0tosBOvK4VyD1am-x","_score":null,"_source":{"@timestamp":"2023-11-09T23:39:21.065237301Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:39:21.064+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1674","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573161065]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jXp0tosBOvK4VyD1gpXK","_score":null,"_source":{"@timestamp":"2023-11-09T23:39:24.066434869Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:39:24.066+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1675","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573164066]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jnp0tosBOvK4VyD1gpXK","_score":null,"_source":{"@timestamp":"2023-11-09T23:39:27.067297233Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:39:27.066+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1676","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573167067]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7np0tosBOvK4VyD1mtjj","_score":null,"_source":{"@timestamp":"2023-11-09T23:39:30.067995966Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:39:30.067+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1677","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573170067]},{"_index":"continuous-tests-pods-2023.11.09","_id":"73p0tosBOvK4VyD1mtjj","_score":null,"_source":{"@timestamp":"2023-11-09T23:39:33.068927818Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:39:33.068+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1678","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573173068]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bHt0tosBOvK4VyD1rhH3","_score":null,"_source":{"@timestamp":"2023-11-09T23:39:36.069491953Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:39:36.069+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1679","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573176069]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bXt0tosBOvK4VyD1rhH3","_score":null,"_source":{"@timestamp":"2023-11-09T23:39:39.070452270Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:39:39.070+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1680","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573179070]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2nt0tosBOvK4VyD1x1IO","_score":null,"_source":{"@timestamp":"2023-11-09T23:39:42.071214447Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:39:42.070+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1681","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573182071]},{"_index":"continuous-tests-pods-2023.11.09","_id":"23t0tosBOvK4VyD1x1IO","_score":null,"_source":{"@timestamp":"2023-11-09T23:39:45.072079562Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:39:45.071+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1682","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573185072]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lHt0tosBOvK4VyD134gk","_score":null,"_source":{"@timestamp":"2023-11-09T23:39:48.072626222Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:39:48.072+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1683","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573188072]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lXt0tosBOvK4VyD134gk","_score":null,"_source":{"@timestamp":"2023-11-09T23:39:51.073377669Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:39:51.073+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1684","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573191073]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Gnt0tosBOvK4VyD196c7","_score":null,"_source":{"@timestamp":"2023-11-09T23:39:54.074414846Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:39:54.074+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1685","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573194074]},{"_index":"continuous-tests-pods-2023.11.09","_id":"G3t0tosBOvK4VyD196c7","_score":null,"_source":{"@timestamp":"2023-11-09T23:39:57.075245089Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:39:57.074+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1686","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573197075]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Mnt1tosBOvK4VyD1D75P","_score":null,"_source":{"@timestamp":"2023-11-09T23:40:00.075705824Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:40:00.075+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1687","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573200075]},{"_index":"continuous-tests-pods-2023.11.09","_id":"M3t1tosBOvK4VyD1D75P","_score":null,"_source":{"@timestamp":"2023-11-09T23:40:03.081601114Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:40:03.081+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1688","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573203081]},{"_index":"continuous-tests-pods-2023.11.09","_id":"83t1tosBOvK4VyD1J9dn","_score":null,"_source":{"@timestamp":"2023-11-09T23:40:06.082738795Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:40:06.082+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1689","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573206082]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9Ht1tosBOvK4VyD1J9dn","_score":null,"_source":{"@timestamp":"2023-11-09T23:40:09.083284791Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:40:09.082+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1690","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573209083]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9Xt1tosBOvK4VyD1J9dn","_score":null,"_source":{"@timestamp":"2023-11-09T23:40:12.084576198Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:40:12.084+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1691","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573212084]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_Xx1tosBOvK4VyD1RxqM","_score":null,"_source":{"@timestamp":"2023-11-09T23:40:15.084715031Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:40:15.084+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1692","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573215084]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_nx1tosBOvK4VyD1RxqM","_score":null,"_source":{"@timestamp":"2023-11-09T23:40:18.085558175Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:40:18.085+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1693","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573218085]},{"_index":"continuous-tests-pods-2023.11.09","_id":"znx1tosBOvK4VyD1X02i","_score":null,"_source":{"@timestamp":"2023-11-09T23:40:21.086706192Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:40:21.086+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1694","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573221086]},{"_index":"continuous-tests-pods-2023.11.09","_id":"z3x1tosBOvK4VyD1X02i","_score":null,"_source":{"@timestamp":"2023-11-09T23:40:24.087891350Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:40:24.087+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1695","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573224087]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SXx1tosBOvK4VyD1d2e7","_score":null,"_source":{"@timestamp":"2023-11-09T23:40:27.088467791Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:40:27.088+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1696","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573227088]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Snx1tosBOvK4VyD1d2e7","_score":null,"_source":{"@timestamp":"2023-11-09T23:40:30.093959665Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:40:30.093+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1697","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573230093]},{"_index":"continuous-tests-pods-2023.11.09","_id":"knx1tosBOvK4VyD1j6DT","_score":null,"_source":{"@timestamp":"2023-11-09T23:40:33.094431787Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:40:33.094+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1698","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573233094]},{"_index":"continuous-tests-pods-2023.11.09","_id":"k3x1tosBOvK4VyD1j6DT","_score":null,"_source":{"@timestamp":"2023-11-09T23:40:36.095354017Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:40:36.094+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1699","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573236095]},{"_index":"continuous-tests-pods-2023.11.09","_id":"knx1tosBOvK4VyD1p_Tr","_score":null,"_source":{"@timestamp":"2023-11-09T23:40:39.095732079Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:40:39.095+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1700","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573239095]},{"_index":"continuous-tests-pods-2023.11.09","_id":"k3x1tosBOvK4VyD1p_Tr","_score":null,"_source":{"@timestamp":"2023-11-09T23:40:42.096582099Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:40:42.096+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1701","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573242096]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9H11tosBOvK4VyD1wEcD","_score":null,"_source":{"@timestamp":"2023-11-09T23:40:45.096812565Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:40:45.096+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1702","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573245096]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9X11tosBOvK4VyD1wEcD","_score":null,"_source":{"@timestamp":"2023-11-09T23:40:48.097311535Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:40:48.096+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1703","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573248097]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kH11tosBOvK4VyD11JAX","_score":null,"_source":{"@timestamp":"2023-11-09T23:40:51.097916218Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:40:51.097+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1704","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573251097]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kX11tosBOvK4VyD11JAX","_score":null,"_source":{"@timestamp":"2023-11-09T23:40:54.099430656Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:40:54.099+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1705","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573254099]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2H11tosBOvK4VyD17OIu","_score":null,"_source":{"@timestamp":"2023-11-09T23:40:57.099976381Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:40:57.099+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1706","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573257099]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2X11tosBOvK4VyD17OIu","_score":null,"_source":{"@timestamp":"2023-11-09T23:41:00.100665667Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:41:00.100+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1707","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573260100]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9312tosBOvK4VyD1BP9I","_score":null,"_source":{"@timestamp":"2023-11-09T23:41:03.101187520Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:41:03.100+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1708","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573263101]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-H12tosBOvK4VyD1BP9I","_score":null,"_source":{"@timestamp":"2023-11-09T23:41:06.102238102Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:41:06.101+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1709","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573266102]},{"_index":"continuous-tests-pods-2023.11.09","_id":"U352tosBOvK4VyD1HBhi","_score":null,"_source":{"@timestamp":"2023-11-09T23:41:09.103347597Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:41:09.103+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1710","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573269103]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VH52tosBOvK4VyD1HBhi","_score":null,"_source":{"@timestamp":"2023-11-09T23:41:12.103941686Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:41:12.103+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1711","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573272103]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tH52tosBOvK4VyD1NDB6","_score":null,"_source":{"@timestamp":"2023-11-09T23:41:15.104953828Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:41:15.104+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1712","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573275104]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tX52tosBOvK4VyD1NDB6","_score":null,"_source":{"@timestamp":"2023-11-09T23:41:18.106244253Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:41:18.105+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1713","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573278106]},{"_index":"continuous-tests-pods-2023.11.09","_id":"P352tosBOvK4VyD1TEmS","_score":null,"_source":{"@timestamp":"2023-11-09T23:41:21.107174674Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:41:21.106+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1714","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573281107]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QH52tosBOvK4VyD1TEmS","_score":null,"_source":{"@timestamp":"2023-11-09T23:41:24.107771502Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:41:24.107+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1715","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573284107]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QX52tosBOvK4VyD1TEmS","_score":null,"_source":{"@timestamp":"2023-11-09T23:41:27.109009109Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:41:27.108+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1716","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573287109]},{"_index":"continuous-tests-pods-2023.11.09","_id":"D352tosBOvK4VyD1bJq5","_score":null,"_source":{"@timestamp":"2023-11-09T23:41:30.110077287Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:41:30.109+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1717","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573290110]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EH52tosBOvK4VyD1bJq5","_score":null,"_source":{"@timestamp":"2023-11-09T23:41:33.111216648Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:41:33.110+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1718","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573293111]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2n52tosBOvK4VyD1hN_S","_score":null,"_source":{"@timestamp":"2023-11-09T23:41:36.112039382Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:41:36.111+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1719","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573296112]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2352tosBOvK4VyD1hN_S","_score":null,"_source":{"@timestamp":"2023-11-09T23:41:39.113455338Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:41:39.113+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1720","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573299113]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DH92tosBOvK4VyD1nCbp","_score":null,"_source":{"@timestamp":"2023-11-09T23:41:42.114353281Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:41:42.113+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1721","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573302114]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DX92tosBOvK4VyD1nCbp","_score":null,"_source":{"@timestamp":"2023-11-09T23:41:45.119745504Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:41:45.119+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1722","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573305119]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1X92tosBOvK4VyD1tVoA","_score":null,"_source":{"@timestamp":"2023-11-09T23:41:48.120273648Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:41:48.119+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1723","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573308120]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1n92tosBOvK4VyD1tVoA","_score":null,"_source":{"@timestamp":"2023-11-09T23:41:51.121488358Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:41:51.121+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1724","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573311121]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QX92tosBOvK4VyD1zXMV","_score":null,"_source":{"@timestamp":"2023-11-09T23:41:54.121957809Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:41:54.121+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1725","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573314121]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Qn92tosBOvK4VyD1zXMV","_score":null,"_source":{"@timestamp":"2023-11-09T23:41:57.123335594Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:41:57.123+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1726","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573317123]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0H92tosBOvK4VyD15Yst","_score":null,"_source":{"@timestamp":"2023-11-09T23:42:00.124454556Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:42:00.124+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1727","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573320124]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0X92tosBOvK4VyD15Yst","_score":null,"_source":{"@timestamp":"2023-11-09T23:42:03.124937602Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:42:03.124+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1728","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573323124]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dn92tosBOvK4VyD1-aFG","_score":null,"_source":{"@timestamp":"2023-11-09T23:42:06.125824257Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:42:06.125+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1729","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573326125]},{"_index":"continuous-tests-pods-2023.11.09","_id":"d392tosBOvK4VyD1-aFG","_score":null,"_source":{"@timestamp":"2023-11-09T23:42:09.127249729Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:42:09.126+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1730","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573329127]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5H93tosBOvK4VyD1EdJf","_score":null,"_source":{"@timestamp":"2023-11-09T23:42:12.128664830Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:42:12.128+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1731","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573332128]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5X93tosBOvK4VyD1EdJf","_score":null,"_source":{"@timestamp":"2023-11-09T23:42:15.128973531Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:42:15.128+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1732","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573335128]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9IB3tosBOvK4VyD1KQ56","_score":null,"_source":{"@timestamp":"2023-11-09T23:42:18.129947383Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:42:18.129+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1733","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573338129]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9YB3tosBOvK4VyD1KQ56","_score":null,"_source":{"@timestamp":"2023-11-09T23:42:21.130805713Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:42:21.130+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1734","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573341130]},{"_index":"continuous-tests-pods-2023.11.09","_id":"c4B3tosBOvK4VyD1QSiQ","_score":null,"_source":{"@timestamp":"2023-11-09T23:42:24.131704604Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:42:24.131+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1735","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573344131]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dIB3tosBOvK4VyD1QSiQ","_score":null,"_source":{"@timestamp":"2023-11-09T23:42:27.132396475Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:42:27.132+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1736","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573347132]},{"_index":"continuous-tests-pods-2023.11.09","_id":"f4B3tosBOvK4VyD1WVWk","_score":null,"_source":{"@timestamp":"2023-11-09T23:42:30.132918606Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:42:30.132+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1737","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573350132]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gIB3tosBOvK4VyD1WVWk","_score":null,"_source":{"@timestamp":"2023-11-09T23:42:33.137758708Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:42:33.137+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1738","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573353137]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VoB3tosBOvK4VyD1ca-9","_score":null,"_source":{"@timestamp":"2023-11-09T23:42:36.138153938Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:42:36.137+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1739","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573356138]},{"_index":"continuous-tests-pods-2023.11.09","_id":"V4B3tosBOvK4VyD1ca-9","_score":null,"_source":{"@timestamp":"2023-11-09T23:42:39.139189290Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:42:39.138+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1740","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573359139]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YoB3tosBOvK4VyD1hfzW","_score":null,"_source":{"@timestamp":"2023-11-09T23:42:42.140557047Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:42:42.140+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1741","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573362140]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Y4B3tosBOvK4VyD1hfzW","_score":null,"_source":{"@timestamp":"2023-11-09T23:42:45.141539401Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:42:45.141+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1742","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573365141]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TYF3tosBOvK4VyD1nVHv","_score":null,"_source":{"@timestamp":"2023-11-09T23:42:48.142101087Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:42:48.141+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1743","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573368142]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ToF3tosBOvK4VyD1nVHv","_score":null,"_source":{"@timestamp":"2023-11-09T23:42:51.142945486Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:42:51.142+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1744","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573371142]},{"_index":"continuous-tests-pods-2023.11.09","_id":"R4F3tosBOvK4VyD1tqYJ","_score":null,"_source":{"@timestamp":"2023-11-09T23:42:54.143666331Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:42:54.143+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1745","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573374143]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SIF3tosBOvK4VyD1tqYJ","_score":null,"_source":{"@timestamp":"2023-11-09T23:42:57.144249822Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:42:57.143+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1746","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573377144]},{"_index":"continuous-tests-pods-2023.11.09","_id":"C4F3tosBOvK4VyD1zsAe","_score":null,"_source":{"@timestamp":"2023-11-09T23:43:00.144718109Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:43:00.144+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1747","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573380144]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DIF3tosBOvK4VyD1zsAe","_score":null,"_source":{"@timestamp":"2023-11-09T23:43:03.145419872Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:43:03.145+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1748","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573383145]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DYF3tosBOvK4VyD1zsAe","_score":null,"_source":{"@timestamp":"2023-11-09T23:43:03.878851210Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:43:03.878+00:00 Cleaning up expired records tid=1 + count=1749","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573383878]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DoF3tosBOvK4VyD1zsAe","_score":null,"_source":{"@timestamp":"2023-11-09T23:43:03.878969393Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:43:03.878+00:00 Cleaned up expired records tid=1 + size=0 count=1750","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573383878]},{"_index":"continuous-tests-pods-2023.11.09","_id":"D4F3tosBOvK4VyD1zsAe","_score":null,"_source":{"@timestamp":"2023-11-09T23:43:03.878996354Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:43:03.878+00:00 Cleaning up query iterator tid=1 + count=1751","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573383878]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EIF3tosBOvK4VyD1zsAe","_score":null,"_source":{"@timestamp":"2023-11-09T23:43:03.879003866Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:43:03.878+00:00 Cleaning up orphaned records tid=1 + count=1752","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573383879]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EYF3tosBOvK4VyD1zsAe","_score":null,"_source":{"@timestamp":"2023-11-09T23:43:03.879085557Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:43:03.878+00:00 Cleaning up query iterator tid=1 + count=1753","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573383879]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WYF3tosBOvK4VyD15th6","_score":null,"_source":{"@timestamp":"2023-11-09T23:43:06.146091923Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:43:06.145+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1754","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573386146]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WoF3tosBOvK4VyD15th6","_score":null,"_source":{"@timestamp":"2023-11-09T23:43:09.147089071Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:43:09.146+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1755","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573389147]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4oF3tosBOvK4VyD1_vKb","_score":null,"_source":{"@timestamp":"2023-11-09T23:43:12.147501175Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:43:12.147+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1756","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573392147]},{"_index":"continuous-tests-pods-2023.11.09","_id":"44F3tosBOvK4VyD1_vKb","_score":null,"_source":{"@timestamp":"2023-11-09T23:43:15.147931298Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:43:15.147+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1757","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573395147]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1oJ4tosBOvK4VyD1Egap","_score":null,"_source":{"@timestamp":"2023-11-09T23:43:18.148713493Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:43:18.148+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1758","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573398148]},{"_index":"continuous-tests-pods-2023.11.09","_id":"14J4tosBOvK4VyD1Egap","_score":null,"_source":{"@timestamp":"2023-11-09T23:43:21.149391965Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:43:21.148+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1759","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573401149]},{"_index":"continuous-tests-pods-2023.11.09","_id":"j4J4tosBOvK4VyD1Kim-","_score":null,"_source":{"@timestamp":"2023-11-09T23:43:24.150552750Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:43:24.150+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1760","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573404150]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kIJ4tosBOvK4VyD1Kim-","_score":null,"_source":{"@timestamp":"2023-11-09T23:43:27.151304731Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:43:27.150+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1761","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573407151]},{"_index":"continuous-tests-pods-2023.11.09","_id":"v4J4tosBOvK4VyD1QmbY","_score":null,"_source":{"@timestamp":"2023-11-09T23:43:30.152069451Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:43:30.151+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1762","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573410152]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wIJ4tosBOvK4VyD1QmbY","_score":null,"_source":{"@timestamp":"2023-11-09T23:43:33.152703853Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:43:33.152+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1763","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573413152]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7YJ4tosBOvK4VyD1Wqzy","_score":null,"_source":{"@timestamp":"2023-11-09T23:43:36.153573716Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:43:36.153+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1764","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573416153]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7oJ4tosBOvK4VyD1Wqzy","_score":null,"_source":{"@timestamp":"2023-11-09T23:43:39.154835220Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:43:39.154+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1765","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573419154]},{"_index":"continuous-tests-pods-2023.11.09","_id":"a4J4tosBOvK4VyD1c-oK","_score":null,"_source":{"@timestamp":"2023-11-09T23:43:42.155410295Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:43:42.155+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1766","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573422155]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bIJ4tosBOvK4VyD1c-oK","_score":null,"_source":{"@timestamp":"2023-11-09T23:43:45.156257809Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:43:45.155+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1767","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573425156]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YoN4tosBOvK4VyD1ixYj","_score":null,"_source":{"@timestamp":"2023-11-09T23:43:48.160912069Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:43:48.160+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1768","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573428160]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Y4N4tosBOvK4VyD1ixYj","_score":null,"_source":{"@timestamp":"2023-11-09T23:43:51.162118145Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:43:51.161+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1769","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573431162]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PIN4tosBOvK4VyD1nzA4","_score":null,"_source":{"@timestamp":"2023-11-09T23:43:54.163090255Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:43:54.162+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1770","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573434163]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PYN4tosBOvK4VyD1nzA4","_score":null,"_source":{"@timestamp":"2023-11-09T23:43:57.164188128Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:43:57.163+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1771","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573437164]},{"_index":"continuous-tests-pods-2023.11.09","_id":"B4N4tosBOvK4VyD1t1NQ","_score":null,"_source":{"@timestamp":"2023-11-09T23:44:00.164672800Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:44:00.164+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1772","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573440164]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CIN4tosBOvK4VyD1t1NQ","_score":null,"_source":{"@timestamp":"2023-11-09T23:44:03.165236171Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:44:03.164+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1773","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573443165]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yYN4tosBOvK4VyD1z2to","_score":null,"_source":{"@timestamp":"2023-11-09T23:44:06.165927773Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:44:06.165+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1774","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573446165]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yoN4tosBOvK4VyD1z2to","_score":null,"_source":{"@timestamp":"2023-11-09T23:44:09.167243832Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:44:09.166+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1775","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573449167]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yIN4tosBOvK4VyD156GD","_score":null,"_source":{"@timestamp":"2023-11-09T23:44:12.168646600Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:44:12.168+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1776","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573452168]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yYN4tosBOvK4VyD156GD","_score":null,"_source":{"@timestamp":"2023-11-09T23:44:15.169594661Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:44:15.169+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1777","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573455169]},{"_index":"continuous-tests-pods-2023.11.09","_id":"x4N4tosBOvK4VyD1_9qZ","_score":null,"_source":{"@timestamp":"2023-11-09T23:44:18.170141885Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:44:18.169+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1778","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573458170]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yIN4tosBOvK4VyD1_9qZ","_score":null,"_source":{"@timestamp":"2023-11-09T23:44:21.175942742Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:44:21.175+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1779","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573461175]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dYN5tosBOvK4VyD1F_Su","_score":null,"_source":{"@timestamp":"2023-11-09T23:44:24.177191441Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:44:24.176+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1780","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573464177]},{"_index":"continuous-tests-pods-2023.11.09","_id":"doN5tosBOvK4VyD1F_Su","_score":null,"_source":{"@timestamp":"2023-11-09T23:44:27.178417931Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:44:27.178+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1781","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573467178]},{"_index":"continuous-tests-pods-2023.11.09","_id":"d4N5tosBOvK4VyD1F_Su","_score":null,"_source":{"@timestamp":"2023-11-09T23:44:30.179466682Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:44:30.179+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1782","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573470179]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JIR5tosBOvK4VyD1N0jT","_score":null,"_source":{"@timestamp":"2023-11-09T23:44:33.180138218Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:44:33.179+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1783","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573473180]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JYR5tosBOvK4VyD1N0jT","_score":null,"_source":{"@timestamp":"2023-11-09T23:44:36.181388301Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:44:36.181+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1784","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573476181]},{"_index":"continuous-tests-pods-2023.11.09","_id":"soR5tosBOvK4VyD1T57u","_score":null,"_source":{"@timestamp":"2023-11-09T23:44:39.182630608Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:44:39.182+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1785","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573479182]},{"_index":"continuous-tests-pods-2023.11.09","_id":"s4R5tosBOvK4VyD1T57u","_score":null,"_source":{"@timestamp":"2023-11-09T23:44:42.183881617Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:44:42.183+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1786","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573482183]},{"_index":"continuous-tests-pods-2023.11.09","_id":"P4R5tosBOvK4VyD1aPUI","_score":null,"_source":{"@timestamp":"2023-11-09T23:44:45.184194681Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:44:45.183+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1787","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573485184]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QIR5tosBOvK4VyD1aPUI","_score":null,"_source":{"@timestamp":"2023-11-09T23:44:48.189551805Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:44:48.189+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1788","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573488189]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mYV5tosBOvK4VyD1gE0f","_score":null,"_source":{"@timestamp":"2023-11-09T23:44:51.190801715Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:44:51.190+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1789","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573491190]},{"_index":"continuous-tests-pods-2023.11.09","_id":"moV5tosBOvK4VyD1gE0g","_score":null,"_source":{"@timestamp":"2023-11-09T23:44:54.191978098Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:44:54.191+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1790","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573494191]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xIV5tosBOvK4VyD1mHs7","_score":null,"_source":{"@timestamp":"2023-11-09T23:44:57.193296592Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:44:57.193+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1791","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573497193]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xYV5tosBOvK4VyD1mHs7","_score":null,"_source":{"@timestamp":"2023-11-09T23:45:00.193863540Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:45:00.193+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1792","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573500193]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6oV5tosBOvK4VyD1sJZO","_score":null,"_source":{"@timestamp":"2023-11-09T23:45:03.194377229Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:45:03.193+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1793","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573503194]},{"_index":"continuous-tests-pods-2023.11.09","_id":"64V5tosBOvK4VyD1sJZO","_score":null,"_source":{"@timestamp":"2023-11-09T23:45:06.195561773Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:45:06.195+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1794","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573506195]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BYV5tosBOvK4VyD1xKtg","_score":null,"_source":{"@timestamp":"2023-11-09T23:45:09.196289223Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:45:09.196+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1795","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573509196]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BoV5tosBOvK4VyD1xKtg","_score":null,"_source":{"@timestamp":"2023-11-09T23:45:12.197432570Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:45:12.197+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1796","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573512197]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-YV5tosBOvK4VyD13MN2","_score":null,"_source":{"@timestamp":"2023-11-09T23:45:15.198473099Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:45:15.197+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1797","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573515198]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-oV5tosBOvK4VyD13MN2","_score":null,"_source":{"@timestamp":"2023-11-09T23:45:18.199746838Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:45:18.199+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1798","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573518199]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VIV5tosBOvK4VyD19NyQ","_score":null,"_source":{"@timestamp":"2023-11-09T23:45:21.201041219Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:45:21.200+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1799","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573521201]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VYV5tosBOvK4VyD19NyQ","_score":null,"_source":{"@timestamp":"2023-11-09T23:45:24.202060106Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:45:24.201+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1800","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573524202]},{"_index":"continuous-tests-pods-2023.11.09","_id":"M4Z6tosBOvK4VyD1DBKj","_score":null,"_source":{"@timestamp":"2023-11-09T23:45:27.203264427Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:45:27.202+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1801","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573527203]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NIZ6tosBOvK4VyD1DBKj","_score":null,"_source":{"@timestamp":"2023-11-09T23:45:30.204697648Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:45:30.204+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1802","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573530204]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zoZ6tosBOvK4VyD1JFO9","_score":null,"_source":{"@timestamp":"2023-11-09T23:45:33.206745706Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:45:33.206+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1803","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573533206]},{"_index":"continuous-tests-pods-2023.11.09","_id":"z4Z6tosBOvK4VyD1JFO9","_score":null,"_source":{"@timestamp":"2023-11-09T23:45:36.207191479Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:45:36.206+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1804","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573536207]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hIZ6tosBOvK4VyD1PKLd","_score":null,"_source":{"@timestamp":"2023-11-09T23:45:39.208283152Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:45:39.207+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1805","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573539208]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hYZ6tosBOvK4VyD1PKLd","_score":null,"_source":{"@timestamp":"2023-11-09T23:45:42.208832692Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:45:42.208+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1806","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573542208]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PYZ6tosBOvK4VyD1UMft","_score":null,"_source":{"@timestamp":"2023-11-09T23:45:45.209895733Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:45:45.209+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1807","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573545209]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PoZ6tosBOvK4VyD1UMft","_score":null,"_source":{"@timestamp":"2023-11-09T23:45:48.211270013Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:45:48.210+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1808","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573548211]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7YZ6tosBOvK4VyD1afkF","_score":null,"_source":{"@timestamp":"2023-11-09T23:45:51.212377025Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:45:51.212+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1809","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573551212]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7oZ6tosBOvK4VyD1afkF","_score":null,"_source":{"@timestamp":"2023-11-09T23:45:54.213454761Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:45:54.213+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1810","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573554213]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2Id6tosBOvK4VyD1gRAd","_score":null,"_source":{"@timestamp":"2023-11-09T23:45:57.213958974Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:45:57.213+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1811","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573557213]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2Yd6tosBOvK4VyD1gRAd","_score":null,"_source":{"@timestamp":"2023-11-09T23:46:00.214734843Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:46:00.214+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1812","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573560214]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UId6tosBOvK4VyD1mSk2","_score":null,"_source":{"@timestamp":"2023-11-09T23:46:03.215483022Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:46:03.215+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1813","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573563215]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UYd6tosBOvK4VyD1mSk2","_score":null,"_source":{"@timestamp":"2023-11-09T23:46:06.216388652Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:46:06.216+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1814","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573566216]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Xod6tosBOvK4VyD1sUhO","_score":null,"_source":{"@timestamp":"2023-11-09T23:46:09.222399479Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:46:09.222+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1815","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573569222]},{"_index":"continuous-tests-pods-2023.11.09","_id":"X4d6tosBOvK4VyD1sUhO","_score":null,"_source":{"@timestamp":"2023-11-09T23:46:12.223368291Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:46:12.223+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1816","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573572223]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3Yd6tosBOvK4VyD1yYVn","_score":null,"_source":{"@timestamp":"2023-11-09T23:46:15.224107619Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:46:15.223+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1817","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573575224]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3od6tosBOvK4VyD1yYVn","_score":null,"_source":{"@timestamp":"2023-11-09T23:46:18.225228778Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:46:18.224+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1818","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573578225]},{"_index":"continuous-tests-pods-2023.11.09","_id":"34d6tosBOvK4VyD1yYVn","_score":null,"_source":{"@timestamp":"2023-11-09T23:46:21.225840590Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:46:21.225+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1819","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573581225]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iYd6tosBOvK4VyD16beG","_score":null,"_source":{"@timestamp":"2023-11-09T23:46:24.227225731Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:46:24.226+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1820","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573584227]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iod6tosBOvK4VyD16beG","_score":null,"_source":{"@timestamp":"2023-11-09T23:46:27.227786323Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:46:27.227+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1821","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573587227]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2Id7tosBOvK4VyD1Adqb","_score":null,"_source":{"@timestamp":"2023-11-09T23:46:30.228542623Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:46:30.228+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1822","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573590228]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2Yd7tosBOvK4VyD1Adqb","_score":null,"_source":{"@timestamp":"2023-11-09T23:46:33.229177683Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:46:33.228+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1823","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573593229]},{"_index":"continuous-tests-pods-2023.11.09","_id":"t4h7tosBOvK4VyD1GS-0","_score":null,"_source":{"@timestamp":"2023-11-09T23:46:36.230463130Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:46:36.230+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1824","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573596230]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uIh7tosBOvK4VyD1GS-0","_score":null,"_source":{"@timestamp":"2023-11-09T23:46:39.231094806Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:46:39.230+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1825","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573599231]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uoh7tosBOvK4VyD1MYTJ","_score":null,"_source":{"@timestamp":"2023-11-09T23:46:42.232425119Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:46:42.231+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1826","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573602232]},{"_index":"continuous-tests-pods-2023.11.09","_id":"u4h7tosBOvK4VyD1MYTJ","_score":null,"_source":{"@timestamp":"2023-11-09T23:46:45.233022760Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:46:45.232+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1827","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573605233]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2Ih7tosBOvK4VyD1Sdnf","_score":null,"_source":{"@timestamp":"2023-11-09T23:46:48.234413837Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:46:48.234+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1828","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573608234]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2Yh7tosBOvK4VyD1Sdnf","_score":null,"_source":{"@timestamp":"2023-11-09T23:46:51.235463344Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:46:51.235+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1829","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573611235]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XYl7tosBOvK4VyD1YS_4","_score":null,"_source":{"@timestamp":"2023-11-09T23:46:54.236889897Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:46:54.236+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1830","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573614236]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Xol7tosBOvK4VyD1YS_4","_score":null,"_source":{"@timestamp":"2023-11-09T23:46:57.237592446Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:46:57.237+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1831","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573617237]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iYl7tosBOvK4VyD1dk8P","_score":null,"_source":{"@timestamp":"2023-11-09T23:47:00.238923783Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:47:00.238+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1832","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573620238]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iol7tosBOvK4VyD1dk8P","_score":null,"_source":{"@timestamp":"2023-11-09T23:47:03.239784436Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:47:03.239+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1833","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573623239]},{"_index":"continuous-tests-pods-2023.11.09","_id":"74l7tosBOvK4VyD1jmgq","_score":null,"_source":{"@timestamp":"2023-11-09T23:47:06.241032903Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:47:06.240+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1834","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573626241]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8Il7tosBOvK4VyD1jmgq","_score":null,"_source":{"@timestamp":"2023-11-09T23:47:09.242092115Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:47:09.241+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1835","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573629242]},{"_index":"continuous-tests-pods-2023.11.09","_id":"44l7tosBOvK4VyD1poE9","_score":null,"_source":{"@timestamp":"2023-11-09T23:47:12.242984054Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:47:12.242+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1836","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573632242]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5Il7tosBOvK4VyD1poE9","_score":null,"_source":{"@timestamp":"2023-11-09T23:47:15.244138930Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:47:15.243+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1837","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573635244]},{"_index":"continuous-tests-pods-2023.11.09","_id":"e4l7tosBOvK4VyD1vppW","_score":null,"_source":{"@timestamp":"2023-11-09T23:47:18.245583620Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:47:18.245+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1838","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573638245]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fIl7tosBOvK4VyD1vppW","_score":null,"_source":{"@timestamp":"2023-11-09T23:47:21.246187751Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:47:21.245+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1839","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573641246]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eIl7tosBOvK4VyD11r5u","_score":null,"_source":{"@timestamp":"2023-11-09T23:47:24.247380529Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:47:24.247+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1840","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573644247]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eYl7tosBOvK4VyD11r5u","_score":null,"_source":{"@timestamp":"2023-11-09T23:47:27.248456518Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:47:27.248+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1841","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573647248]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XIp7tosBOvK4VyD17gGH","_score":null,"_source":{"@timestamp":"2023-11-09T23:47:30.249344123Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:47:30.249+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1842","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573650249]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XYp7tosBOvK4VyD17gGH","_score":null,"_source":{"@timestamp":"2023-11-09T23:47:33.250559369Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:47:33.250+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1843","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573653250]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Xop7tosBOvK4VyD17gGH","_score":null,"_source":{"@timestamp":"2023-11-09T23:47:36.251591646Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:47:36.251+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1844","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573656251]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-Yp8tosBOvK4VyD1Dmmo","_score":null,"_source":{"@timestamp":"2023-11-09T23:47:39.252480702Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:47:39.252+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1845","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573659252]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-op8tosBOvK4VyD1Dmmo","_score":null,"_source":{"@timestamp":"2023-11-09T23:47:42.253929290Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:47:42.253+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1846","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573662253]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vIp8tosBOvK4VyD1Jp_G","_score":null,"_source":{"@timestamp":"2023-11-09T23:47:45.259044212Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:47:45.258+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1847","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573665259]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vYp8tosBOvK4VyD1Jp_G","_score":null,"_source":{"@timestamp":"2023-11-09T23:47:48.260187618Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:47:48.259+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1848","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573668260]},{"_index":"continuous-tests-pods-2023.11.09","_id":"q4p8tosBOvK4VyD1Pszg","_score":null,"_source":{"@timestamp":"2023-11-09T23:47:51.261711019Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:47:51.261+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1849","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573671261]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rIp8tosBOvK4VyD1Pszg","_score":null,"_source":{"@timestamp":"2023-11-09T23:47:54.262069213Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:47:54.261+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1850","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573674262]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sYp8tosBOvK4VyD1Vub3","_score":null,"_source":{"@timestamp":"2023-11-09T23:47:57.263539341Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:47:57.263+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1851","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573677263]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sop8tosBOvK4VyD1Vub3","_score":null,"_source":{"@timestamp":"2023-11-09T23:48:00.264021748Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:48:00.263+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1852","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573680264]},{"_index":"continuous-tests-pods-2023.11.09","_id":"P4p8tosBOvK4VyD1b_4N","_score":null,"_source":{"@timestamp":"2023-11-09T23:48:03.264648382Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:48:03.264+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1853","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573683264]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QIp8tosBOvK4VyD1b_4N","_score":null,"_source":{"@timestamp":"2023-11-09T23:48:03.879652999Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:48:03.879+00:00 Cleaning up expired records tid=1 + count=1854","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573683879]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QYp8tosBOvK4VyD1b_4N","_score":null,"_source":{"@timestamp":"2023-11-09T23:48:03.879696671Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:48:03.879+00:00 Cleaned up expired records tid=1 + size=0 count=1855","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573683879]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Qop8tosBOvK4VyD1b_4N","_score":null,"_source":{"@timestamp":"2023-11-09T23:48:03.879732707Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:48:03.879+00:00 Cleaning up query iterator tid=1 + count=1856","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573683879]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Q4p8tosBOvK4VyD1b_4N","_score":null,"_source":{"@timestamp":"2023-11-09T23:48:03.879786197Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:48:03.879+00:00 Cleaning up orphaned records tid=1 + count=1857","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573683879]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RIp8tosBOvK4VyD1b_4N","_score":null,"_source":{"@timestamp":"2023-11-09T23:48:03.880212446Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:48:03.879+00:00 Cleaning up query iterator tid=1 + count=1858","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573683880]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RYp8tosBOvK4VyD1b_4N","_score":null,"_source":{"@timestamp":"2023-11-09T23:48:06.265322079Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:48:06.264+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1859","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573686265]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8It8tosBOvK4VyD1hx8l","_score":null,"_source":{"@timestamp":"2023-11-09T23:48:09.265917524Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:48:09.265+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1860","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573689265]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8Yt8tosBOvK4VyD1hx8l","_score":null,"_source":{"@timestamp":"2023-11-09T23:48:12.266385717Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:48:12.266+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1861","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573692266]},{"_index":"continuous-tests-pods-2023.11.09","_id":"b4t8tosBOvK4VyD1m1Q6","_score":null,"_source":{"@timestamp":"2023-11-09T23:48:15.266929027Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:48:15.266+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1862","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573695266]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cIt8tosBOvK4VyD1m1Q6","_score":null,"_source":{"@timestamp":"2023-11-09T23:48:18.271926967Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:48:18.271+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1863","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573698271]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Dot8tosBOvK4VyD1s4lQ","_score":null,"_source":{"@timestamp":"2023-11-09T23:48:21.273110590Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:48:21.272+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1864","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573701273]},{"_index":"continuous-tests-pods-2023.11.09","_id":"D4t8tosBOvK4VyD1s4lQ","_score":null,"_source":{"@timestamp":"2023-11-09T23:48:24.273650309Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:48:24.273+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1865","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573704273]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6Yt8tosBOvK4VyD1y6Vn","_score":null,"_source":{"@timestamp":"2023-11-09T23:48:27.274305983Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:48:27.273+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1866","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573707274]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6ot8tosBOvK4VyD1y6Vn","_score":null,"_source":{"@timestamp":"2023-11-09T23:48:30.275639490Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:48:30.275+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1867","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573710275]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6ot8tosBOvK4VyD14-Z_","_score":null,"_source":{"@timestamp":"2023-11-09T23:48:33.276838811Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:48:33.276+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1868","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573713276]},{"_index":"continuous-tests-pods-2023.11.09","_id":"64t8tosBOvK4VyD14-Z_","_score":null,"_source":{"@timestamp":"2023-11-09T23:48:36.277835832Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:48:36.277+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1869","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573716277]},{"_index":"continuous-tests-pods-2023.11.09","_id":"R4x8tosBOvK4VyD1-0GV","_score":null,"_source":{"@timestamp":"2023-11-09T23:48:39.282778784Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:48:39.282+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1870","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573719282]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SIx8tosBOvK4VyD1-0GV","_score":null,"_source":{"@timestamp":"2023-11-09T23:48:42.283381331Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:48:42.283+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1871","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573722283]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hIx9tosBOvK4VyD1E56t","_score":null,"_source":{"@timestamp":"2023-11-09T23:48:45.284337682Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:48:45.283+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1872","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573725284]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hYx9tosBOvK4VyD1E56t","_score":null,"_source":{"@timestamp":"2023-11-09T23:48:48.284868849Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:48:48.284+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1873","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573728284]},{"_index":"continuous-tests-pods-2023.11.09","_id":"44x9tosBOvK4VyD1J-jB","_score":null,"_source":{"@timestamp":"2023-11-09T23:48:51.286130748Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:48:51.285+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1874","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573731286]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5Ix9tosBOvK4VyD1J-jB","_score":null,"_source":{"@timestamp":"2023-11-09T23:48:54.287511101Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:48:54.287+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1875","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573734287]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_o19tosBOvK4VyD1PzfZ","_score":null,"_source":{"@timestamp":"2023-11-09T23:48:57.288214462Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:48:57.287+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1876","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573737288]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_419tosBOvK4VyD1PzfZ","_score":null,"_source":{"@timestamp":"2023-11-09T23:49:00.289550547Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:49:00.289+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1877","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573740289]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6o19tosBOvK4VyD1V1fx","_score":null,"_source":{"@timestamp":"2023-11-09T23:49:03.290311885Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:49:03.289+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1878","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573743290]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6419tosBOvK4VyD1V1fx","_score":null,"_source":{"@timestamp":"2023-11-09T23:49:06.291355246Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:49:06.290+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1879","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573746291]},{"_index":"continuous-tests-pods-2023.11.09","_id":"J419tosBOvK4VyD1cHcN","_score":null,"_source":{"@timestamp":"2023-11-09T23:49:09.292334932Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:49:09.292+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1880","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573749292]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KI19tosBOvK4VyD1cHcN","_score":null,"_source":{"@timestamp":"2023-11-09T23:49:12.293328627Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:49:12.292+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1881","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573752293]},{"_index":"continuous-tests-pods-2023.11.09","_id":"T419tosBOvK4VyD1iJQo","_score":null,"_source":{"@timestamp":"2023-11-09T23:49:15.294385033Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:49:15.294+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1882","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573755294]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UI19tosBOvK4VyD1iJQo","_score":null,"_source":{"@timestamp":"2023-11-09T23:49:18.295906962Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:49:18.295+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1883","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573758295]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HI19tosBOvK4VyD1oLFI","_score":null,"_source":{"@timestamp":"2023-11-09T23:49:21.301123999Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:49:21.300+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1884","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573761301]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HY19tosBOvK4VyD1oLFI","_score":null,"_source":{"@timestamp":"2023-11-09T23:49:24.302486543Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:49:24.302+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1885","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573764302]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ho19tosBOvK4VyD1oLFI","_score":null,"_source":{"@timestamp":"2023-11-09T23:49:27.303046537Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:49:27.302+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1886","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573767303]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Oo59tosBOvK4VyD1wANi","_score":null,"_source":{"@timestamp":"2023-11-09T23:49:30.304264100Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:49:30.303+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1887","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573770304]},{"_index":"continuous-tests-pods-2023.11.09","_id":"O459tosBOvK4VyD1wANi","_score":null,"_source":{"@timestamp":"2023-11-09T23:49:33.305046065Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:49:33.304+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1888","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573773305]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Yo59tosBOvK4VyD12Ex-","_score":null,"_source":{"@timestamp":"2023-11-09T23:49:36.306014042Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:49:36.305+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1889","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573776306]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Y459tosBOvK4VyD12Ex-","_score":null,"_source":{"@timestamp":"2023-11-09T23:49:39.307541483Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:49:39.307+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1890","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573779307]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GI59tosBOvK4VyD18JiV","_score":null,"_source":{"@timestamp":"2023-11-09T23:49:42.308783354Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:49:42.308+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1891","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573782308]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GY59tosBOvK4VyD18JiV","_score":null,"_source":{"@timestamp":"2023-11-09T23:49:45.309972201Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:49:45.309+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1892","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573785309]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7o5-tosBOvK4VyD1CNOp","_score":null,"_source":{"@timestamp":"2023-11-09T23:49:48.314938432Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:49:48.314+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1893","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573788314]},{"_index":"continuous-tests-pods-2023.11.09","_id":"745-tosBOvK4VyD1CNOp","_score":null,"_source":{"@timestamp":"2023-11-09T23:49:51.316347636Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:49:51.315+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1894","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573791316]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TI5-tosBOvK4VyD1IPXD","_score":null,"_source":{"@timestamp":"2023-11-09T23:49:54.317272690Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:49:54.316+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1895","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573794317]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TY5-tosBOvK4VyD1IPXD","_score":null,"_source":{"@timestamp":"2023-11-09T23:49:57.318381682Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:49:57.317+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1896","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573797318]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EY9-tosBOvK4VyD1OBHa","_score":null,"_source":{"@timestamp":"2023-11-09T23:50:00.318979194Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:50:00.318+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1897","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573800318]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Eo9-tosBOvK4VyD1OBHa","_score":null,"_source":{"@timestamp":"2023-11-09T23:50:03.319927565Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:50:03.319+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1898","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573803319]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fY9-tosBOvK4VyD1TCnz","_score":null,"_source":{"@timestamp":"2023-11-09T23:50:06.320806863Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:50:06.320+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1899","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573806320]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fo9-tosBOvK4VyD1TCnz","_score":null,"_source":{"@timestamp":"2023-11-09T23:50:09.322122291Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:50:09.321+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1900","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573809322]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VI9-tosBOvK4VyD1ZVkO","_score":null,"_source":{"@timestamp":"2023-11-09T23:50:12.323315203Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:50:12.322+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1901","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573812323]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VY9-tosBOvK4VyD1ZVkO","_score":null,"_source":{"@timestamp":"2023-11-09T23:50:15.324115749Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:50:15.323+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1902","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573815324]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fo9-tosBOvK4VyD1fZUi","_score":null,"_source":{"@timestamp":"2023-11-09T23:50:18.325226043Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:50:18.324+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1903","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573818325]},{"_index":"continuous-tests-pods-2023.11.09","_id":"f49-tosBOvK4VyD1fZUi","_score":null,"_source":{"@timestamp":"2023-11-09T23:50:21.325847046Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:50:21.325+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1904","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573821325]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EY9-tosBOvK4VyD1lbw7","_score":null,"_source":{"@timestamp":"2023-11-09T23:50:24.326694621Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:50:24.326+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1905","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573824326]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Eo9-tosBOvK4VyD1lbw7","_score":null,"_source":{"@timestamp":"2023-11-09T23:50:27.327829144Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:50:27.327+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1906","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573827327]},{"_index":"continuous-tests-pods-2023.11.09","_id":"go9-tosBOvK4VyD1rd9R","_score":null,"_source":{"@timestamp":"2023-11-09T23:50:30.328619162Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:50:30.328+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1907","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573830328]},{"_index":"continuous-tests-pods-2023.11.09","_id":"g49-tosBOvK4VyD1rd9R","_score":null,"_source":{"@timestamp":"2023-11-09T23:50:33.329927021Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:50:33.329+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1908","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573833329]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tZB-tosBOvK4VyD1xTRn","_score":null,"_source":{"@timestamp":"2023-11-09T23:50:36.334760514Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:50:36.334+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1909","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573836334]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tpB-tosBOvK4VyD1xTRn","_score":null,"_source":{"@timestamp":"2023-11-09T23:50:39.335388966Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:50:39.335+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1910","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573839335]},{"_index":"continuous-tests-pods-2023.11.09","_id":"t5B-tosBOvK4VyD1xTRn","_score":null,"_source":{"@timestamp":"2023-11-09T23:50:42.336766373Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:50:42.336+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1911","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573842336]},{"_index":"continuous-tests-pods-2023.11.09","_id":"w5B-tosBOvK4VyD15a2N","_score":null,"_source":{"@timestamp":"2023-11-09T23:50:45.337658233Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:50:45.337+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1912","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573845337]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xJB-tosBOvK4VyD15a2N","_score":null,"_source":{"@timestamp":"2023-11-09T23:50:48.338425672Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:50:48.338+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1913","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573848338]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ipF-tosBOvK4VyD1_Qek","_score":null,"_source":{"@timestamp":"2023-11-09T23:50:51.339915363Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:50:51.339+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1914","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573851339]},{"_index":"continuous-tests-pods-2023.11.09","_id":"i5F-tosBOvK4VyD1_Qek","_score":null,"_source":{"@timestamp":"2023-11-09T23:50:54.341387939Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:50:54.341+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1915","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573854341]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aZF_tosBOvK4VyD1FVm-","_score":null,"_source":{"@timestamp":"2023-11-09T23:50:57.342792283Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:50:57.342+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1916","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573857342]},{"_index":"continuous-tests-pods-2023.11.09","_id":"apF_tosBOvK4VyD1FVm-","_score":null,"_source":{"@timestamp":"2023-11-09T23:51:00.343983329Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:51:00.343+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1917","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573860343]},{"_index":"continuous-tests-pods-2023.11.09","_id":"u5F_tosBOvK4VyD1LXzP","_score":null,"_source":{"@timestamp":"2023-11-09T23:51:03.349907394Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:51:03.349+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1918","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573863349]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vJF_tosBOvK4VyD1LXzP","_score":null,"_source":{"@timestamp":"2023-11-09T23:51:06.350950128Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:51:06.350+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1919","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573866350]},{"_index":"continuous-tests-pods-2023.11.09","_id":"j5F_tosBOvK4VyD1RZvp","_score":null,"_source":{"@timestamp":"2023-11-09T23:51:09.352480671Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:51:09.352+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1920","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573869352]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kJF_tosBOvK4VyD1RZvp","_score":null,"_source":{"@timestamp":"2023-11-09T23:51:12.353688006Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:51:12.353+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1921","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573872353]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7ZF_tosBOvK4VyD1Xbj-","_score":null,"_source":{"@timestamp":"2023-11-09T23:51:15.354703290Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:51:15.354+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1922","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573875354]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7pF_tosBOvK4VyD1Xbj-","_score":null,"_source":{"@timestamp":"2023-11-09T23:51:18.355040346Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:51:18.354+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1923","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573878355]},{"_index":"continuous-tests-pods-2023.11.09","_id":"t5F_tosBOvK4VyD1ctIU","_score":null,"_source":{"@timestamp":"2023-11-09T23:51:21.356311357Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:51:21.355+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1924","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573881356]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uJF_tosBOvK4VyD1ctIU","_score":null,"_source":{"@timestamp":"2023-11-09T23:51:24.357508567Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:51:24.357+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1925","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573884357]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Q5F_tosBOvK4VyD1iv4r","_score":null,"_source":{"@timestamp":"2023-11-09T23:51:27.358212451Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:51:27.357+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1926","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573887358]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RJF_tosBOvK4VyD1iv4r","_score":null,"_source":{"@timestamp":"2023-11-09T23:51:30.359378912Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:51:30.358+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1927","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573890359]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FJJ_tosBOvK4VyD1okNH","_score":null,"_source":{"@timestamp":"2023-11-09T23:51:33.359949451Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:51:33.359+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1928","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573893359]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FZJ_tosBOvK4VyD1okNH","_score":null,"_source":{"@timestamp":"2023-11-09T23:51:36.361104609Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:51:36.360+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1929","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573896361]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YpJ_tosBOvK4VyD1uoll","_score":null,"_source":{"@timestamp":"2023-11-09T23:51:39.362283251Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:51:39.361+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1930","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573899362]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Y5J_tosBOvK4VyD1uoll","_score":null,"_source":{"@timestamp":"2023-11-09T23:51:42.363815824Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:51:42.363+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1931","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573902363]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iZJ_tosBOvK4VyD10uV7","_score":null,"_source":{"@timestamp":"2023-11-09T23:51:45.369290506Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:51:45.369+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1932","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573905369]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ipJ_tosBOvK4VyD10uV7","_score":null,"_source":{"@timestamp":"2023-11-09T23:51:48.370592349Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:51:48.370+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1933","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573908370]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NZN_tosBOvK4VyD16guT","_score":null,"_source":{"@timestamp":"2023-11-09T23:51:51.371610611Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:51:51.371+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1934","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573911371]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NpN_tosBOvK4VyD16guT","_score":null,"_source":{"@timestamp":"2023-11-09T23:51:54.372464527Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:51:54.372+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1935","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573914372]},{"_index":"continuous-tests-pods-2023.11.09","_id":"E5N_tosBOvK4VyD1_iSl","_score":null,"_source":{"@timestamp":"2023-11-09T23:51:57.373877480Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:51:57.373+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1936","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573917373]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FJN_tosBOvK4VyD1_iSl","_score":null,"_source":{"@timestamp":"2023-11-09T23:52:00.375495134Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:52:00.375+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1937","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573920375]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1ZOAtosBOvK4VyD1FkLA","_score":null,"_source":{"@timestamp":"2023-11-09T23:52:03.376468275Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:52:03.376+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1938","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573923376]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1pOAtosBOvK4VyD1FkLA","_score":null,"_source":{"@timestamp":"2023-11-09T23:52:06.377797108Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:52:06.377+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1939","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573926377]},{"_index":"continuous-tests-pods-2023.11.09","_id":"F5OAtosBOvK4VyD1LmHa","_score":null,"_source":{"@timestamp":"2023-11-09T23:52:09.378998908Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:52:09.378+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1940","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573929378]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GJOAtosBOvK4VyD1LmHa","_score":null,"_source":{"@timestamp":"2023-11-09T23:52:12.384647763Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:52:12.384+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1941","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573932384]},{"_index":"continuous-tests-pods-2023.11.09","_id":"A5OAtosBOvK4VyD1Rpz0","_score":null,"_source":{"@timestamp":"2023-11-09T23:52:15.386077481Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:52:15.385+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1942","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573935386]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BJOAtosBOvK4VyD1Rpz0","_score":null,"_source":{"@timestamp":"2023-11-09T23:52:18.386974108Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:52:18.386+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1943","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573938386]},{"_index":"continuous-tests-pods-2023.11.09","_id":"V5OAtosBOvK4VyD1X9UM","_score":null,"_source":{"@timestamp":"2023-11-09T23:52:21.388054742Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:52:21.387+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1944","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573941388]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WJOAtosBOvK4VyD1X9UM","_score":null,"_source":{"@timestamp":"2023-11-09T23:52:24.389326142Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:52:24.389+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1945","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573944389]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IpOAtosBOvK4VyD1d_Ii","_score":null,"_source":{"@timestamp":"2023-11-09T23:52:27.390610927Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:52:27.390+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1946","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573947390]},{"_index":"continuous-tests-pods-2023.11.09","_id":"I5OAtosBOvK4VyD1d_Ii","_score":null,"_source":{"@timestamp":"2023-11-09T23:52:30.391871618Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:52:30.391+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1947","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573950391]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JJOAtosBOvK4VyD1d_Ii","_score":null,"_source":{"@timestamp":"2023-11-09T23:52:33.393137575Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:52:33.392+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1948","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573953393]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qZSAtosBOvK4VyD1lz9X","_score":null,"_source":{"@timestamp":"2023-11-09T23:52:36.394728247Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:52:36.394+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1949","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573956394]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qpSAtosBOvK4VyD1lz9X","_score":null,"_source":{"@timestamp":"2023-11-09T23:52:39.395964594Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:52:39.395+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1950","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573959395]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TpSAtosBOvK4VyD1r5he","_score":null,"_source":{"@timestamp":"2023-11-09T23:52:42.397476299Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:52:42.397+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1951","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573962397]},{"_index":"continuous-tests-pods-2023.11.09","_id":"T5SAtosBOvK4VyD1r5he","_score":null,"_source":{"@timestamp":"2023-11-09T23:52:45.402633770Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:52:45.402+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1952","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573965402]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OZSAtosBOvK4VyD1x-92","_score":null,"_source":{"@timestamp":"2023-11-09T23:52:48.403408293Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:52:48.403+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1953","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573968403]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OpSAtosBOvK4VyD1x-92","_score":null,"_source":{"@timestamp":"2023-11-09T23:52:51.404454976Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:52:51.404+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1954","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573971404]},{"_index":"continuous-tests-pods-2023.11.09","_id":"25WAtosBOvK4VyD130WO","_score":null,"_source":{"@timestamp":"2023-11-09T23:52:54.405395590Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:52:54.405+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1955","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573974405]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3JWAtosBOvK4VyD130WO","_score":null,"_source":{"@timestamp":"2023-11-09T23:52:57.406504849Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:52:57.406+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1956","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573977406]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_5WAtosBOvK4VyD194mo","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:00.407495892Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:00.407+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1957","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573980407]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AJWAtosBOvK4VyD194qo","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:03.408708586Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:03.408+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1958","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573983408]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AZWAtosBOvK4VyD194qo","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:03.856112452Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:03.855+00:00 Begin iterating blocks... topics=\"codex + discoveryengine\" tid=1 count=1959","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573983856]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ApWAtosBOvK4VyD194qo","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:03.856149063Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:03.855+00:00 Iterating blocks finished. topics=\"codex + discoveryengine\" tid=1 count=1960","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573983856]},{"_index":"continuous-tests-pods-2023.11.09","_id":"A5WAtosBOvK4VyD194qo","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:03.856154799Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:03.855+00:00 About to sleep advertise loop topics=\"codex + discoveryengine\" tid=1 sleep=30m count=1961","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573983856]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BJWAtosBOvK4VyD194qo","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:03.880571246Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:03.880+00:00 Cleaning up expired records tid=1 + count=1962","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573983880]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BZWAtosBOvK4VyD194qo","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:03.880606254Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:03.880+00:00 Cleaned up expired records tid=1 + size=0 count=1963","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573983880]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BpWAtosBOvK4VyD194qo","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:03.880631347Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:03.880+00:00 Cleaning up query iterator tid=1 + count=1964","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573983880]},{"_index":"continuous-tests-pods-2023.11.09","_id":"B5WAtosBOvK4VyD194qo","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:03.880640454Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:03.880+00:00 Cleaning up orphaned records tid=1 + count=1965","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573983880]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CJWAtosBOvK4VyD194qo","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:03.880651003Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:03.880+00:00 Cleaning up query iterator tid=1 + count=1966","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573983880]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Z5WBtosBOvK4VyD1D6TA","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:06.409682093Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:06.409+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1967","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573986409]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aJWBtosBOvK4VyD1D6TA","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:09.410639287Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:09.410+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1968","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573989410]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RpWBtosBOvK4VyD1I7vX","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:12.411373942Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:12.411+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1969","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573992411]},{"_index":"continuous-tests-pods-2023.11.09","_id":"R5WBtosBOvK4VyD1I7vX","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:15.412862293Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:15.412+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1970","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573995412]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JJWBtosBOvK4VyD1O9Xv","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:18.413750196Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:18.413+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1971","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699573998413]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JZWBtosBOvK4VyD1O9Xv","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:21.414772955Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:21.414+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1972","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574001414]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VZWBtosBOvK4VyD1VPIH","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:24.415610027Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:24.415+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1973","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574004415]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VpWBtosBOvK4VyD1VPIH","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:27.416956089Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:27.416+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1974","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574007416]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hJaBtosBOvK4VyD1bDoe","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:30.418483044Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:30.418+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1975","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574010418]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hZaBtosBOvK4VyD1bDoe","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:33.424116683Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:33.423+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1976","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574013424]},{"_index":"continuous-tests-pods-2023.11.09","_id":"i5aBtosBOvK4VyD1hIM0","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:36.424741629Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:36.424+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1977","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574016424]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jJaBtosBOvK4VyD1hIM0","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:39.426330843Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:39.426+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1978","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574019426]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bZaBtosBOvK4VyD1nM9Z","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:42.427616207Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:42.427+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1979","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574022427]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bpaBtosBOvK4VyD1nM9Z","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:45.428412915Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:45.428+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1980","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574025428]},{"_index":"continuous-tests-pods-2023.11.09","_id":"b5aBtosBOvK4VyD1nM9Z","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:48.429250226Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:48.428+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1981","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574028429]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NZeBtosBOvK4VyD1vA9s","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:51.430391479Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:51.430+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1982","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574031430]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NpeBtosBOvK4VyD1vA9s","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:54.436106944Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:54.435+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1983","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574034436]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wJeBtosBOvK4VyD11CeF","_score":null,"_source":{"@timestamp":"2023-11-09T23:53:57.437742246Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:53:57.437+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1984","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574037437]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wZeBtosBOvK4VyD11CeF","_score":null,"_source":{"@timestamp":"2023-11-09T23:54:00.438618976Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:54:00.438+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1985","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574040438]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OJeBtosBOvK4VyD17EO1","_score":null,"_source":{"@timestamp":"2023-11-09T23:54:03.439866779Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:54:03.439+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1986","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574043439]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OZeBtosBOvK4VyD17EO1","_score":null,"_source":{"@timestamp":"2023-11-09T23:54:06.440741345Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:54:06.440+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1987","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574046440]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AJeCtosBOvK4VyD1BF-0","_score":null,"_source":{"@timestamp":"2023-11-09T23:54:09.441521643Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:54:09.441+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1988","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574049441]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AZeCtosBOvK4VyD1BF-0","_score":null,"_source":{"@timestamp":"2023-11-09T23:54:12.442433526Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:54:12.442+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1989","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574052442]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kpeCtosBOvK4VyD1HJjN","_score":null,"_source":{"@timestamp":"2023-11-09T23:54:15.443474474Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:54:15.443+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1990","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574055443]},{"_index":"continuous-tests-pods-2023.11.09","_id":"k5eCtosBOvK4VyD1HJjN","_score":null,"_source":{"@timestamp":"2023-11-09T23:54:18.444522927Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:54:18.444+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1991","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574058444]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NpeCtosBOvK4VyD1NNDm","_score":null,"_source":{"@timestamp":"2023-11-09T23:54:21.445481860Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:54:21.445+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1992","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574061445]},{"_index":"continuous-tests-pods-2023.11.09","_id":"N5eCtosBOvK4VyD1NNDm","_score":null,"_source":{"@timestamp":"2023-11-09T23:54:24.446998333Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:54:24.446+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1993","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574064446]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TJeCtosBOvK4VyD1SOr5","_score":null,"_source":{"@timestamp":"2023-11-09T23:54:27.447899563Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:54:27.447+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1994","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574067447]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TZeCtosBOvK4VyD1SOr5","_score":null,"_source":{"@timestamp":"2023-11-09T23:54:30.449259333Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:54:30.448+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1995","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574070449]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LpiCtosBOvK4VyD1YQwU","_score":null,"_source":{"@timestamp":"2023-11-09T23:54:33.450612502Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:54:33.450+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1996","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574073450]},{"_index":"continuous-tests-pods-2023.11.09","_id":"L5iCtosBOvK4VyD1YQwU","_score":null,"_source":{"@timestamp":"2023-11-09T23:54:36.451376142Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:54:36.451+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1997","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574076451]},{"_index":"continuous-tests-pods-2023.11.09","_id":"l5iCtosBOvK4VyD1eV4m","_score":null,"_source":{"@timestamp":"2023-11-09T23:54:39.452432265Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:54:39.452+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1998","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574079452]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mJiCtosBOvK4VyD1eV4m","_score":null,"_source":{"@timestamp":"2023-11-09T23:54:42.453155031Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:54:42.452+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=1999","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574082453]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cZiCtosBOvK4VyD1kbQ8","_score":null,"_source":{"@timestamp":"2023-11-09T23:54:45.454246447Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:54:45.453+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2000","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574085454]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cpiCtosBOvK4VyD1kbQ8","_score":null,"_source":{"@timestamp":"2023-11-09T23:54:48.455149078Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:54:48.454+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2001","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574088455]},{"_index":"continuous-tests-pods-2023.11.09","_id":"d5mCtosBOvK4VyD1qQNT","_score":null,"_source":{"@timestamp":"2023-11-09T23:54:51.456716958Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:54:51.456+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2002","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574091456]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eJmCtosBOvK4VyD1qQNT","_score":null,"_source":{"@timestamp":"2023-11-09T23:54:54.457811145Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:54:54.457+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2003","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574094457]},{"_index":"continuous-tests-pods-2023.11.09","_id":"K5mCtosBOvK4VyD1wUdp","_score":null,"_source":{"@timestamp":"2023-11-09T23:54:57.458988162Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:54:57.458+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2004","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574097458]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LJmCtosBOvK4VyD1wUdp","_score":null,"_source":{"@timestamp":"2023-11-09T23:55:00.459831535Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:55:00.459+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2005","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574100459]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LZmCtosBOvK4VyD1wUdp","_score":null,"_source":{"@timestamp":"2023-11-09T23:55:03.461338988Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:55:03.461+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2006","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574103461]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BpmCtosBOvK4VyD14ZqU","_score":null,"_source":{"@timestamp":"2023-11-09T23:55:06.467558930Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:55:06.467+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2007","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574106467]},{"_index":"continuous-tests-pods-2023.11.09","_id":"B5mCtosBOvK4VyD14ZqU","_score":null,"_source":{"@timestamp":"2023-11-09T23:55:09.468818818Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:55:09.468+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2008","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574109468]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MZmCtosBOvK4VyD1-bSu","_score":null,"_source":{"@timestamp":"2023-11-09T23:55:12.470496052Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:55:12.470+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2009","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574112470]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MpmCtosBOvK4VyD1-bSu","_score":null,"_source":{"@timestamp":"2023-11-09T23:55:15.472361708Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:55:15.471+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2010","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574115472]},{"_index":"continuous-tests-pods-2023.11.09","_id":"i5mDtosBOvK4VyD1Ec_G","_score":null,"_source":{"@timestamp":"2023-11-09T23:55:18.472559544Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:55:18.472+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2011","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574118472]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jJmDtosBOvK4VyD1Ec_G","_score":null,"_source":{"@timestamp":"2023-11-09T23:55:21.474114506Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:55:21.473+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2012","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574121474]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NpmDtosBOvK4VyD1Kevd","_score":null,"_source":{"@timestamp":"2023-11-09T23:55:24.475473783Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:55:24.475+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2013","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574124475]},{"_index":"continuous-tests-pods-2023.11.09","_id":"N5mDtosBOvK4VyD1Kevd","_score":null,"_source":{"@timestamp":"2023-11-09T23:55:27.476736205Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:55:27.476+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2014","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574127476]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3pqDtosBOvK4VyD1QSr2","_score":null,"_source":{"@timestamp":"2023-11-09T23:55:30.478568848Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:55:30.478+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2015","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574130478]},{"_index":"continuous-tests-pods-2023.11.09","_id":"35qDtosBOvK4VyD1QSr2","_score":null,"_source":{"@timestamp":"2023-11-09T23:55:33.479631701Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:55:33.479+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2016","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574133479]},{"_index":"continuous-tests-pods-2023.11.09","_id":"h5qDtosBOvK4VyD1WnIL","_score":null,"_source":{"@timestamp":"2023-11-09T23:55:36.481146573Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:55:36.480+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2017","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574136481]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iJqDtosBOvK4VyD1WnIL","_score":null,"_source":{"@timestamp":"2023-11-09T23:55:39.482634884Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:55:39.482+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2018","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574139482]},{"_index":"continuous-tests-pods-2023.11.09","_id":"l5qDtosBOvK4VyD1brMg","_score":null,"_source":{"@timestamp":"2023-11-09T23:55:42.483682931Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:55:42.483+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2019","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574142483]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mJqDtosBOvK4VyD1brMg","_score":null,"_source":{"@timestamp":"2023-11-09T23:55:45.484731695Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:55:45.484+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2020","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574145484]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7ZqDtosBOvK4VyD1ht9G","_score":null,"_source":{"@timestamp":"2023-11-09T23:55:48.486076964Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:55:48.485+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2021","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574148486]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7pqDtosBOvK4VyD1ht9G","_score":null,"_source":{"@timestamp":"2023-11-09T23:55:51.487725280Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:55:51.487+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2022","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574151487]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SJuDtosBOvK4VyD1nhVV","_score":null,"_source":{"@timestamp":"2023-11-09T23:55:54.489334809Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:55:54.489+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2023","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574154489]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SZuDtosBOvK4VyD1nhVV","_score":null,"_source":{"@timestamp":"2023-11-09T23:55:57.490760795Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:55:57.490+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2024","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574157490]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qpuDtosBOvK4VyD1tjFo","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:00.491866380Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:56:00.491+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2025","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574160491]},{"_index":"continuous-tests-pods-2023.11.09","_id":"q5uDtosBOvK4VyD1tjFo","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:03.492930528Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:56:03.492+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2026","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574163492]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VJuDtosBOvK4VyD1zk1_","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:06.494038011Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:56:06.493+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2027","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574166494]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VZuDtosBOvK4VyD1zk1_","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:09.494735839Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:56:09.494+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2028","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574169494]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1JuDtosBOvK4VyD15niZ","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:12.496318666Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:56:12.496+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2029","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574172496]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1ZuDtosBOvK4VyD15niZ","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:15.497620466Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:56:15.497+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2030","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574175497]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4ZuDtosBOvK4VyD1-qyv","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:18.498656915Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:56:18.498+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2031","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574178498]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4puDtosBOvK4VyD1-qyv","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:21.500019687Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:56:21.499+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2032","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574181500]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0ZuEtosBOvK4VyD1EtjE","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:24.501013602Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:56:24.500+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2033","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574184501]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0puEtosBOvK4VyD1EtjE","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:27.502356932Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:56:27.502+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2034","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574187502]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-5uEtosBOvK4VyD1KvLa","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:30.503496647Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:56:30.503+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2035","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574190503]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_JuEtosBOvK4VyD1KvLa","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:33.504845852Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:56:33.504+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2036","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574193504]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FpyEtosBOvK4VyD1QlDz","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:36.506039409Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:56:36.505+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2037","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574196506]},{"_index":"continuous-tests-pods-2023.11.09","_id":"F5yEtosBOvK4VyD1QlDz","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:38.696597371Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 23:56:38.696+00:00 Received request topics=\"codex\" + tid=1 peer=10.244.0.212:42744 meth=GET uri=/api/codex/v1/debug/info count=2038","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574198696]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GJyEtosBOvK4VyD1QlDz","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:38.696665445Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 23:56:38.696+00:00 Serving API request topics=\"codex\" + tid=1 peer=10.244.0.212:42744 meth=GET uri=/api/codex/v1/debug/info path_params=\"(table: + {:})\" query_params=\"(table: {:})\" content_body=none(ContentBody) count=2039","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574198696]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GZyEtosBOvK4VyD1QlDz","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:38.697009229Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 23:56:38.696+00:00 Received response from handler topics=\"codex\" + tid=1 status=200 meth=GET peer=10.244.0.212:42744 uri=/api/codex/v1/debug/info + content_type=application/json content_size=2435 count=2040","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574198697]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GpyEtosBOvK4VyD1QlDz","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:38.729054017Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 23:56:38.728+00:00 Received request topics=\"codex\" + tid=1 peer=10.244.0.212:42754 meth=GET uri=/api/codex/v1/debug/info count=2041","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574198729]},{"_index":"continuous-tests-pods-2023.11.09","_id":"G5yEtosBOvK4VyD1QlDz","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:38.729139770Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 23:56:38.728+00:00 Serving API request topics=\"codex\" + tid=1 peer=10.244.0.212:42754 meth=GET uri=/api/codex/v1/debug/info path_params=\"(table: + {:})\" query_params=\"(table: {:})\" content_body=none(ContentBody) count=2042","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574198729]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HJyEtosBOvK4VyD1QlDz","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:38.729457130Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 23:56:38.729+00:00 Received response from handler topics=\"codex\" + tid=1 status=200 meth=GET peer=10.244.0.212:42754 uri=/api/codex/v1/debug/info + content_type=application/json content_size=2435 count=2043","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574198729]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HZyEtosBOvK4VyD1QlDz","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:39.430496225Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 23:56:39.430+00:00 Received request topics=\"codex\" + tid=1 peer=10.244.0.212:42758 meth=GET uri=/api/codex/v1/debug/info count=2044","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574199430]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HpyEtosBOvK4VyD1QlDz","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:39.430535169Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 23:56:39.430+00:00 Serving API request topics=\"codex\" + tid=1 peer=10.244.0.212:42758 meth=GET uri=/api/codex/v1/debug/info path_params=\"(table: + {:})\" query_params=\"(table: {:})\" content_body=none(ContentBody) count=2045","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574199430]},{"_index":"continuous-tests-pods-2023.11.09","_id":"H5yEtosBOvK4VyD1QlDz","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:39.430863281Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 23:56:39.430+00:00 Received response from handler topics=\"codex\" + tid=1 status=200 meth=GET peer=10.244.0.212:42758 uri=/api/codex/v1/debug/info + content_type=application/json content_size=2435 count=2046","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574199430]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IJyEtosBOvK4VyD1QlDz","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:39.506622437Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:56:39.506+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2047","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574199506]},{"_index":"continuous-tests-pods-2023.11.09","_id":"e5yEtosBOvK4VyD1Wb4Q","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:42.507225170Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:56:42.506+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2048","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574202507]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fJyEtosBOvK4VyD1Wb4Q","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:45.507532233Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:56:45.507+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2049","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574205507]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XJ2EtosBOvK4VyD1cTEw","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:48.508165821Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:56:48.507+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2050","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574208508]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XZ2EtosBOvK4VyD1cTEw","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:51.508474851Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:56:51.508+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2051","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574211508]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XZ2EtosBOvK4VyD1iYZA","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:54.509907013Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:56:54.509+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2052","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574214509]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Xp2EtosBOvK4VyD1iYZA","_score":null,"_source":{"@timestamp":"2023-11-09T23:56:57.511264791Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:56:57.510+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2053","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574217511]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IZ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:00.512293384Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:00.511+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=0 count=2054","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574220512]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ip2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.046713408Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 23:57:03.046+00:00 Received request topics=\"codex\" + tid=1 peer=10.244.0.212:42588 meth=GET uri=/api/codex/v1/download/zdj7Wg6hDGgA9mwKLwE6tEW8vL74iqN4kcfKqNXpRfAzEaj4b + count=2055","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223046]},{"_index":"continuous-tests-pods-2023.11.09","_id":"I52EtosBOvK4VyD1k6It","_score":null,"_ignored":["message.keyword"],"_source":{"@timestamp":"2023-11-09T23:57:03.046763900Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 23:57:03.046+00:00 Serving API request topics=\"codex\" + tid=1 peer=10.244.0.212:42588 meth=GET uri=/api/codex/v1/download/zdj7Wg6hDGgA9mwKLwE6tEW8vL74iqN4kcfKqNXpRfAzEaj4b + path_params=\"(table: {\\\"id\\\": @[\\\"zdj7Wg6hDGgA9mwKLwE6tEW8vL74iqN4kcfKqNXpRfAzEaj4b\\\"]})\" + query_params=\"(table: {:})\" content_body=none(ContentBody) count=2056","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223046]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JJ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.046913877Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.046+00:00 Retrieving manifest for cid topics=\"codex + node\" tid=1 cid=zdj*zEaj4b count=2057","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223046]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JZ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.046934882Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.046+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zdj*zEaj4b count=2058","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223046]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Jp2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.047210952Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.046+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zdj*zEaj4b count=2059","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223047]},{"_index":"continuous-tests-pods-2023.11.09","_id":"J52EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.047261729Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.047+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zdj*zEaj4b peers=0 count=2060","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223047]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KJ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.047268772Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.047+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zdj*zEaj4b inFlight=false count=2061","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223047]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KZ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.047409364Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.047+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zdj*zEaj4b count=2062","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223047]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Kp2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.047431134Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.047+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=2063","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223047]},{"_index":"continuous-tests-pods-2023.11.09","_id":"K52EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.047440421Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.047+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=2064","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223047]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LJ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.047449856Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.047+00:00 No peers to request blocks from. Queue discovery... + topics=\"codex blockexcengine\" tid=1 cid=zdj*zEaj4b count=2065","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223047]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LZ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.047466182Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.047+00:00 Queueing find block topics=\"codex + discoveryengine\" tid=1 cid=zdj*zEaj4b queue=0 count=2066","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223047]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Lp2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.047471034Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.047+00:00 Current number of peers for block topics=\"codex + discoveryengine\" tid=1 cid=zdj*zEaj4b peers=0 count=2067","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223047]},{"_index":"continuous-tests-pods-2023.11.09","_id":"L52EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.047578384Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.047+00:00 Discovering block topics=\"codex + discoveryengine\" tid=1 cid=zdj*zEaj4b count=2068","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223047]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MJ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.047584084Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.047+00:00 Finding providers for block topics=\"codex + discovery\" tid=1 cid=zdj*zEaj4b count=2069","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223047]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MZ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.052074937Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.051+00:00 Discovered peers for block topics=\"codex + discoveryengine\" tid=1 peers=1 cid=zdj*zEaj4b count=2070","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223052]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Mp2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.052108246Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 23:57:03.051+00:00 Dialing peer topics=\"libp2p + dialer\" tid=1 peerId=16U*pYW64F count=2071","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223052]},{"_index":"continuous-tests-pods-2023.11.09","_id":"M52EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.052134156Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.052+00:00 Dialing address topics=\"libp2p + dialer\" tid=1 address=/ip4/10.244.2.250/tcp/8082 peerId=16U*pYW64F hostname=10.244.2.250 + count=2072","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223052]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NJ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.057258413Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.056+00:00 Upgrading connection topics=\"libp2p + muxedupgrade\" tid=1 conn=:654d71cf50209d7c997ccedc direction=Out count=2073","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NZ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.057302654Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.056+00:00 initiating handshake topics=\"libp2p + upgrade\" tid=1 conn=:654d71cf50209d7c997ccedc codec=/multistream/1.0.0 count=2074","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Np2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.057308073Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.057+00:00 selecting proto topics=\"libp2p + upgrade\" tid=1 conn=:654d71cf50209d7c997ccedc proto=/noise count=2075","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"N52EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.059453041Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.059+00:00 multistream handshake success topics=\"libp2p + upgrade\" tid=1 conn=:654d71cf50209d7c997ccedc count=2076","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223059]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OJ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.060073878Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.059+00:00 reading first requested proto topics=\"libp2p + upgrade\" tid=1 conn=:654d71cf50209d7c997ccedc count=2077","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223060]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OZ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.060098532Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.059+00:00 successfully selected topics=\"libp2p + upgrade\" tid=1 conn=:654d71cf50209d7c997ccedc proto=/noise count=2078","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223060]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Op2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.060103354Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.059+00:00 Securing connection topics=\"libp2p + upgrade\" tid=1 conn=:654d71cf50209d7c997ccedc codec=/noise count=2079","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223060]},{"_index":"continuous-tests-pods-2023.11.09","_id":"O52EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.063519716Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.063+00:00 Muxing connection topics=\"libp2p + muxedupgrade\" tid=1 conn=16U*pYW64F:654d71cf50209d7c997ccedd count=2080","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223063]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PJ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.063577765Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.063+00:00 initiating handshake topics=\"libp2p + upgrade\" tid=1 conn=16U*pYW64F:654d71cf50209d7c997ccedd codec=/multistream/1.0.0 + count=2081","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223063]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PZ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.063582917Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.063+00:00 selecting proto topics=\"libp2p + upgrade\" tid=1 conn=16U*pYW64F:654d71cf50209d7c997ccedd proto=/mplex/6.7.0 + count=2082","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223063]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Pp2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.107783741Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.107+00:00 multistream handshake success topics=\"libp2p + upgrade\" tid=1 conn=16U*pYW64F:654d71cf50209d7c997ccedd count=2083","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223107]},{"_index":"continuous-tests-pods-2023.11.09","_id":"P52EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.152568334Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.152+00:00 reading first requested proto topics=\"libp2p + upgrade\" tid=1 conn=16U*pYW64F:654d71cf50209d7c997ccedd count=2084","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223152]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QJ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.152610075Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.152+00:00 successfully selected topics=\"libp2p + upgrade\" tid=1 conn=16U*pYW64F:654d71cf50209d7c997ccedd proto=/mplex/6.7.0 + count=2085","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223152]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QZ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.152616252Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.152+00:00 Found a muxer topics=\"libp2p + muxedupgrade\" tid=1 conn=16U*pYW64F:654d71cf50209d7c997ccedd muxerName=/mplex/6.7.0 + count=2086","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223152]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Qp2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.152836780Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.152+00:00 Starting stream handler tid=1 + conn=16U*pYW64F:654d71cf50209d7c997ccedf count=2087","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223152]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Q52EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.153073497Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.152+00:00 Upgraded connection topics=\"libp2p + muxedupgrade\" tid=1 conn=16U*pYW64F:654d71cf50209d7c997ccedc sconn=16U*pYW64F:654d71cf50209d7c997ccedd + direction=Out count=2088","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223153]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RJ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.153175897Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 23:57:03.153+00:00 Dial successful topics=\"libp2p + dialer\" tid=1 peerId=16U*pYW64F count=2089","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223153]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RZ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.153416154Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 23:57:03.153+00:00 Created new blockexc peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2090","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223153]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Rp2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.153438388Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.153+00:00 Setting up new peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F count=2091","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223153]},{"_index":"continuous-tests-pods-2023.11.09","_id":"R52EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.153443296Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.153+00:00 Adding peer to peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2092","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223153]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SJ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.153447946Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.153+00:00 Added peer topics=\"codex + blockexcengine\" tid=1 peers=1 count=2093","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223153]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SZ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.153630019Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.153+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantHave items=1 count=2094","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223153]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Sp2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.153652408Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.153+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2095","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223153]},{"_index":"continuous-tests-pods-2023.11.09","_id":"S52EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.153657870Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.153+00:00 Dialing (existing) topics=\"libp2p + dialer\" tid=1 peerId=16U*pYW64F protos=\"@[\\\"/codex/blockexc/1.0.0\\\"]\" + count=2096","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223153]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TJ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.153920795Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.153+00:00 Negotiating stream topics=\"libp2p + dialer\" tid=1 conn=16U*pYW64F:654d71cf50209d7c997ccee0 protos=\"@[\\\"/codex/blockexc/1.0.0\\\"]\" + count=2097","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223153]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TZ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.154287451Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.154+00:00 handling identify request topics=\"libp2p + identify\" tid=1 conn=16U*pYW64F:654d71cf50209d7c997ccedf count=2098","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223154]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Tp2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.154397436Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.154+00:00 exiting identify handler topics=\"libp2p + identify\" tid=1 conn=16U*pYW64F:654d71cf50209d7c997ccedf count=2099","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223154]},{"_index":"continuous-tests-pods-2023.11.09","_id":"T52EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.244847407Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.244+00:00 initiating identify topics=\"libp2p + identify\" tid=1 conn=16U*pYW64F:654d71cf50209d7c997ccee1 count=2100","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223244]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UJ2EtosBOvK4VyD1k6It","_score":null,"_ignored":["message.keyword"],"_source":{"@timestamp":"2023-11-09T23:57:03.245216542Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"DBG + 2023-11-09 23:57:03.245+00:00 decodeMsg: decoded identify topics=\"libp2p + identify\" tid=1 pubkey=some(s...afc0)) addresses=/ip4/0.0.0.0/tcp/8082 protocols=/ipfs/id/1.0.0,/codex/blockexc/1.0.0 + observable_address=some(/ip4/10.244.3.62/tcp/52444) proto_version=ipfs/0.1.0 + agent_version=Codex signedPeerRecord=Some count=2101","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223245]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UZ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.245643915Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.245+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2102","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223245]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Up2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.245915674Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.245+00:00 Stream handler done tid=1 + conn=16U*pYW64F:654d71cf50209d7c997ccedf count=2103","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223245]},{"_index":"continuous-tests-pods-2023.11.09","_id":"U52EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.293053823Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.292+00:00 Starting stream handler tid=1 + conn=16U*pYW64F:654d71cf50209d7c997ccee2 count=2104","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223293]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VJ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.379746437Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.379+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2105","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223379]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VZ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.379827036Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.379+00:00 Handling block presence for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2106","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223379]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Vp2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.379833176Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.379+00:00 Received presence update for peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F blocks=1 count=2107","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223379]},{"_index":"continuous-tests-pods-2023.11.09","_id":"V52EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.379837553Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.379+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2108","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223379]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WJ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.379848977Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.379+00:00 Updating precense topics=\"codex + blockexcengine\" tid=1 have=true price=0 cid=zdj*zEaj4b count=2109","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223379]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WZ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.380781507Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.379+00:00 Peer has blocks in our wantList topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F wantCount=1 count=2110","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223380]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Wp2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.380813048Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.380+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zdj*zEaj4b count=2111","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223380]},{"_index":"continuous-tests-pods-2023.11.09","_id":"W52EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.380819715Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.380+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=2112","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223380]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XJ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.380825366Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.380+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2113","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223380]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XZ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.380848359Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.380+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2114","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223380]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Xp2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.389507552Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.389+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2115","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223389]},{"_index":"continuous-tests-pods-2023.11.09","_id":"X52EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.389547865Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.389+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2116","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223389]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YJ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.390171752Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.389+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=2117","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223390]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YZ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.390270647Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.390+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/manifests/4b/zdj7Wg6hDGgA9mwKLwE6tEW8vL74iqN4kcfKqNXpRfAzEaj4b + cid=zdj*zEaj4b count=2118","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223390]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Yp2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.390798367Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.390+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=51332 cid=zdj*zEaj4b count=2119","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223390]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Y52EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.399709649Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.399+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2120","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223399]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZJ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.399774422Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.399+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zdj*zEaj4b count=2121","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223399]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZZ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.399813120Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.399+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=352553 count=2122","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223399]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Zp2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.399817677Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.399+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=2123","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223399]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Z52EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.399826593Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.399+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zdj*zEaj4b queue=0 count=2124","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223399]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aJ2EtosBOvK4VyD1k6It","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.399970697Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.399+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2125","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223399]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aZ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.399983678Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.399+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2126","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223399]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ap2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.400018582Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.399+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=2127","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223400]},{"_index":"continuous-tests-pods-2023.11.09","_id":"a52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.400196748Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.400+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zdj*zEaj4b count=2128","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223400]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bJ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.400468151Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.400+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zdj*zEaj4b inflight=1 count=2129","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223400]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bZ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.400584602Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.400+00:00 Decoding manifest for cid topics=\"codex + node\" tid=1 cid=zdj*zEaj4b count=2130","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223400]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bp2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.404181989Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.403+00:00 Decoded manifest topics=\"codex + node\" tid=1 cid=zdj*zEaj4b count=2131","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223404]},{"_index":"continuous-tests-pods-2023.11.09","_id":"b52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.404217893Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.403+00:00 Retrieving blocks from manifest topics=\"codex + node\" tid=1 cid=zdj*zEaj4b count=2132","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223404]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cJ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.404270809Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.404+00:00 Creating store stream for manifest topics=\"codex + node\" tid=1 cid=zdj*zEaj4b count=2133","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223404]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cZ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.404724262Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.404+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=2134","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223404]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cp2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.404752356Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.404+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*WUHAaD count=2135","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223404]},{"_index":"continuous-tests-pods-2023.11.09","_id":"c52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.404912692Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.404+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*WUHAaD count=2136","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223404]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dJ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.404923324Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.404+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*WUHAaD peers=1 count=2137","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223404]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dZ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.404929373Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.404+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*WUHAaD inFlight=false count=2138","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223404]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dp2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.405029937Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.404+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*WUHAaD count=2139","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223405]},{"_index":"continuous-tests-pods-2023.11.09","_id":"d52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.405038268Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.404+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=2140","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223405]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eJ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.405049312Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.405+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=2141","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223405]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eZ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.405092873Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.405+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*WUHAaD count=2142","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223405]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ep2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.405130836Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.405+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*WUHAaD inFlight=true count=2143","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223405]},{"_index":"continuous-tests-pods-2023.11.09","_id":"e52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.405224792Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.405+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*WUHAaD count=2144","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223405]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fJ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.405276196Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.405+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=2145","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223405]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fZ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.405318364Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.405+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2146","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223405]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fp2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.405325192Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.405+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2147","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223405]},{"_index":"continuous-tests-pods-2023.11.09","_id":"f52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.405525266Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.405+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*WUHAaD count=2148","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223405]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gJ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.405697237Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.405+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=2149","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223405]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gZ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.405819946Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.405+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zdj*zEaj4b inflight=0 count=2150","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223405]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gp2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.426406658Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.425+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2151","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223426]},{"_index":"continuous-tests-pods-2023.11.09","_id":"g52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.426451780Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.426+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2152","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223426]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hJ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.427319120Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.427+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=2153","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223427]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hZ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.427769995Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.427+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/aD/zb2rhXj8zw4dETp1t8vUPkp3kZkHjsQ4EpvbnFQqfdqWUHAaD + cid=zb2*WUHAaD count=2154","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223427]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hp2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.427914103Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.427+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=116804 cid=zb2*WUHAaD count=2155","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223427]},{"_index":"continuous-tests-pods-2023.11.09","_id":"h52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.433601856Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.433+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2156","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223433]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iJ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.433634739Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.433+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*WUHAaD count=2157","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223433]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iZ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.433639940Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.433+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=28616 count=2158","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223433]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ip2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.433650708Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.433+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=2159","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223433]},{"_index":"continuous-tests-pods-2023.11.09","_id":"i52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.433698724Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.433+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*WUHAaD queue=0 count=2160","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223433]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jJ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.433753723Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.433+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2161","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223433]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jZ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.433762312Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.433+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2162","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223433]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jp2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.433804814Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.433+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=2163","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223433]},{"_index":"continuous-tests-pods-2023.11.09","_id":"j52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.434098952Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.433+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*WUHAaD count=2164","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223434]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kJ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.434270678Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.434+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*WUHAaD inflight=1 count=2165","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223434]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kZ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.434288005Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.434+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*WUHAaD count=2166","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223434]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kp2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.434626743Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.434+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=0 cid=zb2*WUHAaD bytes=65472 blockOffset=0 count=2167","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223434]},{"_index":"continuous-tests-pods-2023.11.09","_id":"k52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.434776609Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.434+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=2168","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223434]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lJ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.435119628Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.434+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=2169","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223435]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lZ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.435134452Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.434+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*MNZBya count=2170","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223435]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lp2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.435168002Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.435+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*MNZBya count=2171","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223435]},{"_index":"continuous-tests-pods-2023.11.09","_id":"l52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.435194674Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.435+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*MNZBya peers=1 count=2172","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223435]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mJ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.435258012Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.435+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*MNZBya inFlight=false count=2173","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223435]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mZ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.435326343Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.435+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*MNZBya count=2174","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223435]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mp2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.435441874Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.435+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=2175","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223435]},{"_index":"continuous-tests-pods-2023.11.09","_id":"m52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.435448829Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.435+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=2176","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223435]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nJ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.435452836Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.435+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*MNZBya count=2177","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223435]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nZ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.435483534Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.435+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*MNZBya inFlight=true count=2178","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223435]},{"_index":"continuous-tests-pods-2023.11.09","_id":"np2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.435653807Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.435+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*MNZBya count=2179","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223435]},{"_index":"continuous-tests-pods-2023.11.09","_id":"n52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.435660339Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.435+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=2180","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223435]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oJ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.435671212Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.435+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2181","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223435]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oZ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.435729932Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.435+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2182","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223435]},{"_index":"continuous-tests-pods-2023.11.09","_id":"op2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.435912029Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.435+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*MNZBya count=2183","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223435]},{"_index":"continuous-tests-pods-2023.11.09","_id":"o52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.436721933Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.436+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=2184","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223436]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pJ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.436774003Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.436+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*WUHAaD inflight=0 count=2185","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223436]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pZ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.450792369Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.450+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2186","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223450]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pp2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.450830834Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.450+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2187","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223450]},{"_index":"continuous-tests-pods-2023.11.09","_id":"p52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.451843505Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.451+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=2188","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223451]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qJ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.451983167Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.451+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/ya/zb2rhjNmkSck8xs4cKUGrQwL5JeekGkkG74nJDVZS3PMNZBya + cid=zb2*MNZBya count=2189","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223451]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qZ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.452481569Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.452+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=182276 cid=zb2*MNZBya count=2190","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223452]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qp2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.458475496Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.458+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2191","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223458]},{"_index":"continuous-tests-pods-2023.11.09","_id":"q52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.458507060Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.458+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*MNZBya count=2192","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223458]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rJ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.458519185Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.458+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=23139 count=2193","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223458]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rZ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.458670196Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.458+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=2194","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223458]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rp2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.458676078Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.458+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*MNZBya queue=0 count=2195","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223458]},{"_index":"continuous-tests-pods-2023.11.09","_id":"r52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.458683637Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.458+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2196","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223458]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sJ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.458735863Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.458+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2197","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223458]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sZ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.458761195Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.458+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=2198","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223458]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sp2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.459047100Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.458+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*MNZBya count=2199","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223459]},{"_index":"continuous-tests-pods-2023.11.09","_id":"s52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.459331618Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.459+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*MNZBya inflight=1 count=2200","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223459]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tJ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.459355547Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.459+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*MNZBya count=2201","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223459]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tZ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.459527885Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.459+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=1 cid=zb2*MNZBya bytes=65472 blockOffset=0 count=2202","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223459]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tp2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.459559845Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.459+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=2203","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223459]},{"_index":"continuous-tests-pods-2023.11.09","_id":"t52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.459834437Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.459+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=2204","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223459]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uJ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.459852992Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.459+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*pn7usJ count=2205","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223459]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uZ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.460015890Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.459+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*pn7usJ count=2206","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223460]},{"_index":"continuous-tests-pods-2023.11.09","_id":"up2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.460029580Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.459+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*pn7usJ peers=1 count=2207","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223460]},{"_index":"continuous-tests-pods-2023.11.09","_id":"u52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.460125057Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.460+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*pn7usJ inFlight=false count=2208","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223460]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vJ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.460289871Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.460+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*pn7usJ count=2209","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223460]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vZ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.460306047Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.460+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=2210","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223460]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vp2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.460312111Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.460+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=2211","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223460]},{"_index":"continuous-tests-pods-2023.11.09","_id":"v52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.460336735Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.460+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*pn7usJ count=2212","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223460]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wJ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.460353626Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.460+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*pn7usJ inFlight=true count=2213","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223460]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wZ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.460457135Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.460+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*pn7usJ count=2214","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223460]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wp2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.460502571Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.460+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=2215","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223460]},{"_index":"continuous-tests-pods-2023.11.09","_id":"w52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.460551783Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.460+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2216","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223460]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xJ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.460562274Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.460+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2217","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223460]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xZ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.460817093Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.460+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*pn7usJ count=2218","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223460]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xp2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.461473091Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.461+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=2219","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223461]},{"_index":"continuous-tests-pods-2023.11.09","_id":"x52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.461565798Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.461+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*MNZBya inflight=0 count=2220","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223461]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yJ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.475663569Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.475+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2221","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223475]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yZ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.475710303Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.475+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2222","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223475]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yp2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.476892147Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.476+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=2223","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223476]},{"_index":"continuous-tests-pods-2023.11.09","_id":"y52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.476943081Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.476+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/sJ/zb2rhkvxgJfxVP8Mc6pM8GwdydqUPwrBmPwEbAxsgfEpn7usJ + cid=zb2*pn7usJ count=2224","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223476]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zJ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.477544426Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.477+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=247748 cid=zb2*pn7usJ count=2225","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223477]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zZ2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.483093837Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.482+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2226","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223483]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zp2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.483130021Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.482+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*pn7usJ count=2227","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223483]},{"_index":"continuous-tests-pods-2023.11.09","_id":"z52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.483137185Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.482+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=22920 count=2228","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223483]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0J2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.483243400Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.483+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=2229","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223483]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0Z2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.483300320Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.483+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*pn7usJ queue=0 count=2230","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223483]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0p2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.483314127Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.483+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2231","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223483]},{"_index":"continuous-tests-pods-2023.11.09","_id":"052EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.483318849Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.483+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2232","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223483]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1J2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.483374260Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.483+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=2233","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223483]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1Z2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.483589344Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.483+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*pn7usJ count=2234","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223483]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1p2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.483759229Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.483+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*pn7usJ inflight=1 count=2235","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223483]},{"_index":"continuous-tests-pods-2023.11.09","_id":"152EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.483772572Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.483+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*pn7usJ count=2236","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223483]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2J2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.483982160Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.483+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=2 cid=zb2*pn7usJ bytes=65472 blockOffset=0 count=2237","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223483]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2Z2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.484042169Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.483+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=2238","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223484]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2p2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.484303611Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.484+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=2239","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223484]},{"_index":"continuous-tests-pods-2023.11.09","_id":"252EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.484339943Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.484+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*gdnrnn count=2240","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223484]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3J2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.484527012Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.484+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*gdnrnn count=2241","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223484]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3Z2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.484578367Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.484+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*gdnrnn peers=1 count=2242","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223484]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3p2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.484585458Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.484+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*gdnrnn inFlight=false count=2243","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223484]},{"_index":"continuous-tests-pods-2023.11.09","_id":"352EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.484803182Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.484+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*gdnrnn count=2244","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223484]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4J2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.484820377Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.484+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=2245","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223484]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4Z2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.484849886Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.484+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=2246","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223484]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4p2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.484863614Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.484+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*gdnrnn count=2247","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223484]},{"_index":"continuous-tests-pods-2023.11.09","_id":"452EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.484869754Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.484+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*gdnrnn inFlight=true count=2248","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223484]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5J2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.484972708Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.484+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*gdnrnn count=2249","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223484]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5Z2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.485044928Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.484+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=2250","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223485]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5p2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.485051379Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.484+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2251","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223485]},{"_index":"continuous-tests-pods-2023.11.09","_id":"552EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.485055320Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.484+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2252","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223485]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6J2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.485292931Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.485+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*gdnrnn count=2253","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223485]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6Z2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.486052096Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.485+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=2254","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223486]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6p2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.486072934Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.485+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*pn7usJ inflight=0 count=2255","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223486]},{"_index":"continuous-tests-pods-2023.11.09","_id":"652EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.499573960Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.499+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2256","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223499]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7J2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.499608846Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.499+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2257","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223499]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7Z2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.500331048Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.500+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=2258","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223500]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7p2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.500541167Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.500+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/nn/zb2rhgCung6DCboa4wn9HMjFYixvtxrFcvDgHK55tn7gdnrnn + cid=zb2*gdnrnn count=2259","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223500]},{"_index":"continuous-tests-pods-2023.11.09","_id":"752EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.501026345Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.500+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=313220 cid=zb2*gdnrnn count=2260","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223501]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8J2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.507284005Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.507+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2261","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223507]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8Z2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.507408509Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.507+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*gdnrnn count=2262","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223507]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8p2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.507495565Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.507+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=22733 count=2263","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223507]},{"_index":"continuous-tests-pods-2023.11.09","_id":"852EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.507516974Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.507+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=2264","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223507]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9J2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.507521114Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.507+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*gdnrnn queue=0 count=2265","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223507]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9Z2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.507565877Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.507+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2266","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223507]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9p2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.507571272Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.507+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2267","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223507]},{"_index":"continuous-tests-pods-2023.11.09","_id":"952EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.507579040Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.507+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=2268","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223507]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-J2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.507743246Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.507+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*gdnrnn count=2269","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223507]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-Z2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.507968165Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.507+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*gdnrnn inflight=1 count=2270","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223507]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-p2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.507985967Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.507+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*gdnrnn count=2271","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223507]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.508272889Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.508+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=3 cid=zb2*gdnrnn bytes=65472 blockOffset=0 count=2272","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223508]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_J2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.508286185Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.508+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=2273","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223508]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_Z2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.508502122Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.508+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=2274","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223508]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_p2EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.508518045Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.508+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*wBmN1q count=2275","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223508]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_52EtosBOvK4VyD1k6Iu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.508681833Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.508+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*wBmN1q count=2276","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223508]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.508721916Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.508+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*wBmN1q peers=1 count=2277","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223508]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.508801224Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.508+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*wBmN1q inFlight=false count=2278","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223508]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ap2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.508826087Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.508+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*wBmN1q count=2279","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223508]},{"_index":"continuous-tests-pods-2023.11.09","_id":"A52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.508870524Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.508+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=2280","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223508]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.508877374Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.508+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=2281","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223508]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.508958418Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.508+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*wBmN1q count=2282","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223508]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Bp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.508968434Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.508+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*wBmN1q inFlight=true count=2283","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223508]},{"_index":"continuous-tests-pods-2023.11.09","_id":"B52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.509087766Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.508+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*wBmN1q count=2284","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223509]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.509112497Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.508+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=2285","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223509]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.509153043Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.509+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2286","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223509]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Cp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.509166964Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.509+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2287","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223509]},{"_index":"continuous-tests-pods-2023.11.09","_id":"C52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.509359432Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.509+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*wBmN1q count=2288","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223509]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.510543998Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.510+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=2289","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223510]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.510570538Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.510+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*gdnrnn inflight=0 count=2290","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223510]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Dp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.512968253Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.512+00:00 About to sleep discovery loop topics=\"codex + discoveryengine\" tid=1 sleep=3s wanted=1 count=2291","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223512]},{"_index":"continuous-tests-pods-2023.11.09","_id":"D52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.513001635Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.512+00:00 Current number of peers for block topics=\"codex + discoveryengine\" tid=1 cid=zb2*wBmN1q peers=0 count=2292","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223513]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.513018993Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.512+00:00 Discovering block topics=\"codex + discoveryengine\" tid=1 cid=zb2*wBmN1q count=2293","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223513]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.513041412Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.512+00:00 Finding providers for block topics=\"codex + discovery\" tid=1 cid=zb2*wBmN1q count=2294","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223513]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ep2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.524366738Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.523+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2295","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223524]},{"_index":"continuous-tests-pods-2023.11.09","_id":"E52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.524406157Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.524+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2296","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223524]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.524883679Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.524+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=2297","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223524]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.525042473Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.524+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/1q/zb2rhc2rMPe6ianGWpd3TBsSQgsjmD2HTS9BJHauuUVwBmN1q + cid=zb2*wBmN1q count=2298","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223525]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Fp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.525508538Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.525+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=378692 cid=zb2*wBmN1q count=2299","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223525]},{"_index":"continuous-tests-pods-2023.11.09","_id":"F52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.530960459Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.530+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2300","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223530]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.531015661Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.530+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*wBmN1q count=2301","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223531]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.531021960Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.530+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=22059 count=2302","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223531]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Gp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.531026714Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.530+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=2303","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223531]},{"_index":"continuous-tests-pods-2023.11.09","_id":"G52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.531040898Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.530+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*wBmN1q queue=0 count=2304","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223531]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.531045478Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.530+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2305","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223531]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.531049720Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.530+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2306","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223531]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Hp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.531095107Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.531+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=2307","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223531]},{"_index":"continuous-tests-pods-2023.11.09","_id":"H52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.531411215Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.531+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*wBmN1q count=2308","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223531]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.531603882Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.531+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*wBmN1q inflight=1 count=2309","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223531]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.531627984Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.531+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*wBmN1q count=2310","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223531]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ip2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.531901716Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.531+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=4 cid=zb2*wBmN1q bytes=65472 blockOffset=0 count=2311","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223531]},{"_index":"continuous-tests-pods-2023.11.09","_id":"I52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.531910177Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.531+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=2312","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223531]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.532114056Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.531+00:00 Discovered peers for block topics=\"codex + discoveryengine\" tid=1 peers=1 cid=zb2*wBmN1q count=2313","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223532]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.532296126Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.532+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=2314","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223532]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Jp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.532305812Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.532+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*kkNvAq count=2315","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223532]},{"_index":"continuous-tests-pods-2023.11.09","_id":"J52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.532577459Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.532+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*kkNvAq count=2316","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223532]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.532586860Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.532+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*kkNvAq peers=1 count=2317","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223532]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.532592309Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.532+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*kkNvAq inFlight=false count=2318","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223532]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Kp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.532602627Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.532+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*kkNvAq count=2319","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223532]},{"_index":"continuous-tests-pods-2023.11.09","_id":"K52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.532719634Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.532+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=2320","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223532]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.532728805Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.532+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=2321","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223532]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.532749348Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.532+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*kkNvAq count=2322","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223532]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Lp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.532852059Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.532+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*kkNvAq inFlight=true count=2323","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223532]},{"_index":"continuous-tests-pods-2023.11.09","_id":"L52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.532865446Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.532+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*kkNvAq count=2324","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223532]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.532956716Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.532+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=2325","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223532]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.533001730Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.532+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2326","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223533]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Mp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.533026780Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.532+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2327","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223533]},{"_index":"continuous-tests-pods-2023.11.09","_id":"M52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.533186044Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.533+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*kkNvAq count=2328","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223533]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.533500604Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.533+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=2329","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223533]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.533586478Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.533+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*wBmN1q inflight=0 count=2330","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223533]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Np2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.550829443Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.550+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2331","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223550]},{"_index":"continuous-tests-pods-2023.11.09","_id":"N52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.550894299Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.550+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2332","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223550]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.551373811Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.551+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=2333","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223551]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.551539882Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.551+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/Aq/zb2rhkyK5zbLhoCtGZhaoDYPwBeKiyh7poRaRKk8kGdkkNvAq + cid=zb2*kkNvAq count=2334","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223551]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Op2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.552080309Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.551+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=444164 cid=zb2*kkNvAq count=2335","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223552]},{"_index":"continuous-tests-pods-2023.11.09","_id":"O52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.557837562Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.557+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2336","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223557]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.557876947Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.557+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*kkNvAq count=2337","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223557]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.557882993Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.557+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=25146 count=2338","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223557]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Pp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.557896669Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.557+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=2339","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223557]},{"_index":"continuous-tests-pods-2023.11.09","_id":"P52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.557905246Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.557+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*kkNvAq queue=0 count=2340","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223557]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.558035152Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.557+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2341","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223558]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.558045175Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.557+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2342","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223558]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Qp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.558055469Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.557+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=2343","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223558]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Q52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.558402616Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.558+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*kkNvAq count=2344","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223558]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.558608826Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.558+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*kkNvAq inflight=1 count=2345","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223558]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.558635983Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.558+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*kkNvAq count=2346","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223558]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Rp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.558830256Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.558+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=5 cid=zb2*kkNvAq bytes=65472 blockOffset=0 count=2347","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223558]},{"_index":"continuous-tests-pods-2023.11.09","_id":"R52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.558843260Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.558+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=2348","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223558]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.559180538Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.559+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=2349","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223559]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.559204018Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.559+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*dYSxYp count=2350","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223559]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Sp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.559364110Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.559+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*dYSxYp count=2351","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223559]},{"_index":"continuous-tests-pods-2023.11.09","_id":"S52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.559372130Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.559+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*dYSxYp peers=1 count=2352","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223559]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.559380930Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.559+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*dYSxYp inFlight=false count=2353","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223559]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.559508626Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.559+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*dYSxYp count=2354","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223559]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Tp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.559515213Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.559+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=2355","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223559]},{"_index":"continuous-tests-pods-2023.11.09","_id":"T52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.559596808Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.559+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=2356","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223559]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.559602957Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.559+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*dYSxYp count=2357","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223559]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.559641536Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.559+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*dYSxYp inFlight=true count=2358","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223559]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Up2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.559769165Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.559+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*dYSxYp count=2359","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223559]},{"_index":"continuous-tests-pods-2023.11.09","_id":"U52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.559780629Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.559+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=2360","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223559]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.559902666Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.559+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2361","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223559]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.559921310Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.559+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2362","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223559]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Vp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.560051865Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.559+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*dYSxYp count=2363","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223560]},{"_index":"continuous-tests-pods-2023.11.09","_id":"V52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.560637483Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.560+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=2364","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223560]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.560756464Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.560+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*kkNvAq inflight=0 count=2365","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223560]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.573024472Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.572+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2366","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223573]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Wp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.573065644Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.572+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2367","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223573]},{"_index":"continuous-tests-pods-2023.11.09","_id":"W52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.573728485Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.573+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=2368","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223573]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.573921591Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.573+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/Yp/zb2rhWj7S7DfCFFphucgHWGwN7AwErDEqCZ3jwDz555dYSxYp + cid=zb2*dYSxYp count=2369","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223573]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.574453619Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.574+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=509636 cid=zb2*dYSxYp count=2370","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223574]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Xp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.580235455Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.579+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2371","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223580]},{"_index":"continuous-tests-pods-2023.11.09","_id":"X52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.580306237Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.580+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*dYSxYp count=2372","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223580]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.580329703Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.580+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=20889 count=2373","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223580]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.580339162Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.580+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=2374","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223580]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Yp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.580376898Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.580+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*dYSxYp queue=0 count=2375","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223580]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Y52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.580443299Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.580+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2376","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223580]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.580639570Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.580+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2377","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223580]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.580655049Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.580+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=2378","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223580]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Zp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.580723775Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.580+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*dYSxYp count=2379","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223580]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Z52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.580940210Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.580+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*dYSxYp inflight=1 count=2380","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223580]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.581015489Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.580+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*dYSxYp count=2381","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223581]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.581256088Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.581+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=6 cid=zb2*dYSxYp bytes=65472 blockOffset=0 count=2382","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223581]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ap2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.581285345Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.581+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=2383","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223581]},{"_index":"continuous-tests-pods-2023.11.09","_id":"a52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.581571526Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.581+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=2384","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223581]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.581618718Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.581+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*dNtfj7 count=2385","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223581]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.581784337Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.581+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*dNtfj7 count=2386","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223581]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.581798936Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.581+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*dNtfj7 peers=1 count=2387","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223581]},{"_index":"continuous-tests-pods-2023.11.09","_id":"b52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.581884634Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.581+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*dNtfj7 inFlight=false count=2388","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223581]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.581921426Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.581+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*dNtfj7 count=2389","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223581]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.581959648Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.581+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=2390","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223581]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.581993558Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.581+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=2391","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223581]},{"_index":"continuous-tests-pods-2023.11.09","_id":"c52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.582108979Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.581+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*dNtfj7 count=2392","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223582]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.582120485Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.582+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*dNtfj7 inFlight=true count=2393","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223582]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.582157715Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.582+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*dNtfj7 count=2394","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223582]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.582179857Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.582+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=2395","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223582]},{"_index":"continuous-tests-pods-2023.11.09","_id":"d52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.582280960Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.582+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2396","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223582]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.582426697Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.582+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2397","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223582]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.582648524Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.582+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*dNtfj7 count=2398","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223582]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ep2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.583768175Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.583+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=2399","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223583]},{"_index":"continuous-tests-pods-2023.11.09","_id":"e52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.583790062Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.583+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*dYSxYp inflight=0 count=2400","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223583]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.588743293Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.588+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2401","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223588]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.588791117Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.588+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2402","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223588]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.589454387Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.589+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=2403","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223589]},{"_index":"continuous-tests-pods-2023.11.09","_id":"f52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.589667843Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.589+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/j7/zb2rhn2z6gTRbm1W1XrLvDXZ2gYWzZt5ze6pZVrKS24dNtfj7 + cid=zb2*dNtfj7 count=2404","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223589]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.590215361Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.589+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=575108 cid=zb2*dNtfj7 count=2405","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223590]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.596470466Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.596+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2406","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223596]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.596527267Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.596+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*dNtfj7 count=2407","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223596]},{"_index":"continuous-tests-pods-2023.11.09","_id":"g52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.596533322Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.596+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=14621 count=2408","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223596]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.596629036Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.596+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=2409","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223596]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.596740157Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.596+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*dNtfj7 queue=0 count=2410","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223596]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.596774301Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.596+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2411","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223596]},{"_index":"continuous-tests-pods-2023.11.09","_id":"h52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.596828274Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.596+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2412","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223596]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.596843167Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.596+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=2413","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223596]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.597140665Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.597+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*dNtfj7 count=2414","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223597]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ip2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.597336049Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.597+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*dNtfj7 inflight=1 count=2415","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223597]},{"_index":"continuous-tests-pods-2023.11.09","_id":"i52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.597370931Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.597+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*dNtfj7 count=2416","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223597]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.597661919Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.597+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=7 cid=zb2*dNtfj7 bytes=65472 blockOffset=0 count=2417","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223597]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.597718259Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.597+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=2418","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223597]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.598139776Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.597+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=2419","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223598]},{"_index":"continuous-tests-pods-2023.11.09","_id":"j52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.598158275Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.597+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*HqFKKb count=2420","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223598]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.598237652Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.598+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*HqFKKb count=2421","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223598]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.598290346Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.598+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*HqFKKb peers=1 count=2422","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223598]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.598403562Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.598+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*HqFKKb inFlight=false count=2423","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223598]},{"_index":"continuous-tests-pods-2023.11.09","_id":"k52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.598590922Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.598+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*HqFKKb count=2424","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223598]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.598617302Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.598+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=2425","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223598]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.598624128Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.598+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=2426","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223598]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.598701377Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.598+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*HqFKKb count=2427","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223598]},{"_index":"continuous-tests-pods-2023.11.09","_id":"l52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.598757805Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.598+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*HqFKKb inFlight=true count=2428","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223598]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.598924014Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.598+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*HqFKKb count=2429","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223598]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.598942784Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.598+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=2430","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223598]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.599010949Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.598+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2431","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223599]},{"_index":"continuous-tests-pods-2023.11.09","_id":"m52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.599082194Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.599+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2432","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223599]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.599261193Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.599+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*HqFKKb count=2433","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223599]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.599634227Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.599+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=2434","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223599]},{"_index":"continuous-tests-pods-2023.11.09","_id":"np2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.599770603Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.599+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*dNtfj7 inflight=0 count=2435","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223599]},{"_index":"continuous-tests-pods-2023.11.09","_id":"n52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.612459455Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.612+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2436","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223612]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.612507502Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.612+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2437","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223612]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.613146519Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.612+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=2438","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223613]},{"_index":"continuous-tests-pods-2023.11.09","_id":"op2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.613424599Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.613+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/Kb/zb2rhki6X7SbYe4KinZyVnzuAVD7CxEMKu78yLSbhzHHqFKKb + cid=zb2*HqFKKb count=2439","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223613]},{"_index":"continuous-tests-pods-2023.11.09","_id":"o52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.613963021Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.613+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=640580 cid=zb2*HqFKKb count=2440","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223613]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.626621109Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.626+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2441","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223626]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.626660156Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.626+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*HqFKKb count=2442","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223626]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.626710516Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.626+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=28343 count=2443","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223626]},{"_index":"continuous-tests-pods-2023.11.09","_id":"p52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.626933529Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.626+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=2444","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223626]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.626969262Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.626+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*HqFKKb queue=0 count=2445","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223626]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.627082405Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.626+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2446","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223627]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.627124771Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.627+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2447","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223627]},{"_index":"continuous-tests-pods-2023.11.09","_id":"q52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.627147396Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.627+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=2448","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223627]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rJ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.627439915Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.627+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*HqFKKb count=2449","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223627]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rZ2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.627644078Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.627+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*HqFKKb inflight=1 count=2450","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223627]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rp2EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.627675295Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.627+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*HqFKKb count=2451","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223627]},{"_index":"continuous-tests-pods-2023.11.09","_id":"r52EtosBOvK4VyD1k6Mu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.627983239Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.627+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=8 cid=zb2*HqFKKb bytes=65472 blockOffset=0 count=2452","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223627]},{"_index":"continuous-tests-pods-2023.11.09","_id":"y52EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.628083019Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.627+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=2453","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223628]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zJ2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.628408847Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.628+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=2454","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223628]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zZ2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.628449290Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.628+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*YA95xF count=2455","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223628]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zp2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.628652693Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.628+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*YA95xF count=2456","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223628]},{"_index":"continuous-tests-pods-2023.11.09","_id":"z52EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.628689028Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.628+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*YA95xF peers=1 count=2457","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223628]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0J2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.628835848Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.628+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*YA95xF inFlight=false count=2458","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223628]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0Z2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.628904197Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.628+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*YA95xF count=2459","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223628]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0p2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.628942071Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.628+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=2460","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223628]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1J2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.629066707Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.629+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*YA95xF count=2462","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223629]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1Z2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.629133068Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.629+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*YA95xF inFlight=true count=2463","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223629]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1p2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.629261832Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.629+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*YA95xF count=2464","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223629]},{"_index":"continuous-tests-pods-2023.11.09","_id":"152EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.629298321Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.629+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=2465","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223629]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2J2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.629384235Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.629+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2466","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223629]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2Z2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.629427182Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.629+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2467","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223629]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2p2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.629619305Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.629+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*YA95xF count=2468","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223629]},{"_index":"continuous-tests-pods-2023.11.09","_id":"252EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.629992594Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.629+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=2469","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223629]},{"_index":"continuous-tests-pods-2023.11.09","_id":"052EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.629005002Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.628+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=2461","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223629]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3J2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.630132505Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.630+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*HqFKKb inflight=0 count=2470","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223630]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3Z2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.643058412Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.642+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2471","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223643]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3p2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.643113085Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.643+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2472","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223643]},{"_index":"continuous-tests-pods-2023.11.09","_id":"352EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.643778022Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.643+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=2473","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223643]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4J2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.644028618Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.643+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/xF/zb2rhd3YuYzfWGwB4Md4t72Q8ew7K8k57sZ2Mq8MBY4YA95xF + cid=zb2*YA95xF count=2474","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223644]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4Z2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.644530267Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.644+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=706052 cid=zb2*YA95xF count=2475","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223644]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4p2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.650946986Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.650+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2476","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223650]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5J2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.651065717Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.650+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=22273 count=2478","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223651]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5Z2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.651302636Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.651+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=2479","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223651]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5p2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.651326964Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.651+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*YA95xF queue=0 count=2480","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223651]},{"_index":"continuous-tests-pods-2023.11.09","_id":"552EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.651436036Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.651+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2481","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223651]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6J2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.651480546Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.651+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2482","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223651]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6Z2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.651678009Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.651+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=2483","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223651]},{"_index":"continuous-tests-pods-2023.11.09","_id":"452EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.651014759Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.650+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*YA95xF count=2477","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223651]},{"_index":"continuous-tests-pods-2023.11.09","_id":"652EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.652133775Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.652+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*YA95xF inflight=1 count=2485","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223652]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7J2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.652148588Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.652+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*YA95xF count=2486","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223652]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7Z2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.652410878Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.652+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=9 cid=zb2*YA95xF bytes=65472 blockOffset=0 count=2487","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223652]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7p2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.652425645Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.652+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=2488","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223652]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8J2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.652662386Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.652+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*KzCtQk count=2490","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223652]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8Z2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.652860420Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.652+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*KzCtQk count=2491","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223652]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8p2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.652874212Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.652+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*KzCtQk peers=1 count=2492","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223652]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6p2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.652019744Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.651+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*YA95xF count=2484","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223652]},{"_index":"continuous-tests-pods-2023.11.09","_id":"752EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.652643952Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.652+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=2489","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223652]},{"_index":"continuous-tests-pods-2023.11.09","_id":"852EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.653133121Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.652+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*KzCtQk inFlight=false count=2493","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223653]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9Z2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.653253043Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.653+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=2495","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223653]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9p2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.653258709Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.653+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=2496","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223653]},{"_index":"continuous-tests-pods-2023.11.09","_id":"952EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.653268338Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.653+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*KzCtQk count=2497","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223653]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-Z2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.653422100Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.653+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*KzCtQk count=2499","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223653]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-p2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.653487820Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.653+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=2500","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223653]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_J2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.653578876Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.653+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2502","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223653]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9J2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.653244028Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.653+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*KzCtQk count=2494","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223653]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-J2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.653293157Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.653+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*KzCtQk inFlight=true count=2498","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223653]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-52EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.653538537Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.653+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2501","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223653]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_Z2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.653733299Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.653+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*KzCtQk count=2503","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223653]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_p2EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.654214517Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.654+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=2504","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223654]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_52EtosBOvK4VyD1lKY-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.654296436Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.654+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*YA95xF inflight=0 count=2505","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223654]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.666223804Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.665+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2506","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223666]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.666266503Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.666+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2507","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223666]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ap2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.666980370Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.666+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=2508","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223666]},{"_index":"continuous-tests-pods-2023.11.09","_id":"A52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.667102290Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.666+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/Qk/zb2rhZFhTP5iRk2DpABgxrTVTW34hdfiYSAWtu9KjRUKzCtQk + cid=zb2*KzCtQk count=2509","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223667]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.667581351Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.667+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=771524 cid=zb2*KzCtQk count=2510","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223667]},{"_index":"continuous-tests-pods-2023.11.09","_id":"C52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.673286712Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.673+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2517","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223673]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.673024548Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.672+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2511","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223673]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Bp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.673109388Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.672+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*KzCtQk count=2512","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223673]},{"_index":"continuous-tests-pods-2023.11.09","_id":"B52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.673117629Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.673+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=20171 count=2513","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223673]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.673188702Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.673+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*KzCtQk queue=0 count=2515","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223673]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.673328197Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.673+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=2518","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223673]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.673477974Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.673+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*KzCtQk count=2519","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223673]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Dp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.673689455Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.673+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*KzCtQk inflight=1 count=2520","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223673]},{"_index":"continuous-tests-pods-2023.11.09","_id":"D52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.673795380Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.673+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*KzCtQk count=2521","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223673]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.673126081Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.673+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=2514","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223673]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Cp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.673235241Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.673+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2516","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223673]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.674263292Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.673+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=10 cid=zb2*KzCtQk bytes=65472 blockOffset=0 count=2522","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223674]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.674309008Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.674+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=2523","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223674]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ep2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.674424359Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.674+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=2524","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223674]},{"_index":"continuous-tests-pods-2023.11.09","_id":"E52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.674530341Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.674+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*uw5Lsi count=2525","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223674]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.674644431Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.674+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*uw5Lsi count=2526","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223674]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.674796117Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.674+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*uw5Lsi peers=1 count=2527","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223674]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Fp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.674929325Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.674+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*uw5Lsi inFlight=false count=2528","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223674]},{"_index":"continuous-tests-pods-2023.11.09","_id":"F52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.674971292Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.674+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*uw5Lsi count=2529","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223674]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.675061873Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.674+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=2530","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223675]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.675078137Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.675+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=2531","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223675]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Gp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.675172225Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.675+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*uw5Lsi count=2532","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223675]},{"_index":"continuous-tests-pods-2023.11.09","_id":"G52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.675189849Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.675+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*uw5Lsi inFlight=true count=2533","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223675]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.675234742Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.675+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*uw5Lsi count=2534","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223675]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.675264890Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.675+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=2535","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223675]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Hp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.675305212Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.675+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2536","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223675]},{"_index":"continuous-tests-pods-2023.11.09","_id":"H52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.675450788Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.675+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2537","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223675]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.675590159Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.675+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*uw5Lsi count=2538","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223675]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.676195905Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.676+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=2539","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223676]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ip2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.676252793Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.676+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*KzCtQk inflight=0 count=2540","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223676]},{"_index":"continuous-tests-pods-2023.11.09","_id":"I52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.688188286Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.687+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2541","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223688]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.688232967Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.688+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2542","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223688]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.688898960Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.688+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=2543","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223688]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Jp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.689158154Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.688+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/si/zb2rhY5reytHE4EnUNhd9ptFcwjBKAUdNXdBej8HWs6uw5Lsi + cid=zb2*uw5Lsi count=2544","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223689]},{"_index":"continuous-tests-pods-2023.11.09","_id":"J52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.689536191Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.689+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=836996 cid=zb2*uw5Lsi count=2545","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223689]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.695451427Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.695+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2551","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223695]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Lp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.695492859Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.695+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2552","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223695]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.695359633Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.694+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2546","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223695]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.695388935Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.695+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*uw5Lsi count=2547","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223695]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Kp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.695394457Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.695+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=20389 count=2548","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223695]},{"_index":"continuous-tests-pods-2023.11.09","_id":"K52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.695441550Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.695+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=2549","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223695]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.695447310Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.695+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*uw5Lsi queue=0 count=2550","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223695]},{"_index":"continuous-tests-pods-2023.11.09","_id":"L52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.695503266Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.695+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=2553","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223695]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.695724784Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.695+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*uw5Lsi count=2554","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223695]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.695925627Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.695+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*uw5Lsi inflight=1 count=2555","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223695]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Mp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.695935046Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.695+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*uw5Lsi count=2556","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223695]},{"_index":"continuous-tests-pods-2023.11.09","_id":"N52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.696703266Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.696+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*4rhgMa count=2561","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223696]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.696714001Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.696+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*4rhgMa peers=1 count=2562","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223696]},{"_index":"continuous-tests-pods-2023.11.09","_id":"M52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.696267336Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.696+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=11 cid=zb2*uw5Lsi bytes=65472 blockOffset=0 count=2557","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223696]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.696291562Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.696+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=2558","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223696]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.696678070Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.696+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=2559","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223696]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Np2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.696697595Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.696+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*4rhgMa count=2560","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223696]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.696721980Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.696+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*4rhgMa inFlight=false count=2563","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223696]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Op2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.696854604Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.696+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*4rhgMa count=2564","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223696]},{"_index":"continuous-tests-pods-2023.11.09","_id":"O52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.696976532Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.696+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=2565","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223696]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.696984302Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.696+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=2566","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223696]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.697003809Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.696+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*4rhgMa count=2567","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223697]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Pp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.697030627Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.696+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*4rhgMa inFlight=true count=2568","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223697]},{"_index":"continuous-tests-pods-2023.11.09","_id":"P52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.697158709Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.697+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*4rhgMa count=2569","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223697]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.697874686Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.697+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*uw5Lsi inflight=0 count=2575","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223697]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.697171640Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.697+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=2570","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223697]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.697250471Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.697+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2571","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223697]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Qp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.697309574Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.697+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2572","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223697]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Q52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.697482787Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.697+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*4rhgMa count=2573","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223697]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.697803478Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.697+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=2574","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223697]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Rp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.710662983Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.710+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2576","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223710]},{"_index":"continuous-tests-pods-2023.11.09","_id":"R52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.710697912Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.710+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2577","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223710]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.711626394Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.711+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=2578","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223711]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.711656940Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.711+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/Ma/zb2rhbViW175zfnoy9XiRSgSVBwdVDHJzyRVbhzXWDE4rhgMa + cid=zb2*4rhgMa count=2579","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223711]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Sp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.712050702Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.711+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=902468 cid=zb2*4rhgMa count=2580","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223712]},{"_index":"continuous-tests-pods-2023.11.09","_id":"S52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.717800952Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.717+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2581","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223717]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.717830750Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.717+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*4rhgMa count=2582","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223717]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.717836110Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.717+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=20985 count=2583","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223717]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Tp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.717847161Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.717+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=2584","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223717]},{"_index":"continuous-tests-pods-2023.11.09","_id":"T52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.717888408Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.717+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*4rhgMa queue=0 count=2585","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223717]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.717940005Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.717+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2586","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223717]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.717975912Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.717+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2587","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223717]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Up2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.718004672Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.717+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=2588","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223718]},{"_index":"continuous-tests-pods-2023.11.09","_id":"U52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.718328618Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.718+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*4rhgMa count=2589","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223718]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.718549748Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.718+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*4rhgMa inflight=1 count=2590","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223718]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.718569128Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.718+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*4rhgMa count=2591","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223718]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Vp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.719716911Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.718+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=12 cid=zb2*4rhgMa bytes=65472 blockOffset=0 count=2592","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223719]},{"_index":"continuous-tests-pods-2023.11.09","_id":"V52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.719737172Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.718+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=2593","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223719]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.719765351Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.719+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=2594","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223719]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.719771915Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.719+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*cixg2M count=2595","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223719]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Wp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.719776625Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.719+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*cixg2M count=2596","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223719]},{"_index":"continuous-tests-pods-2023.11.09","_id":"W52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.719780854Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.719+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*cixg2M peers=1 count=2597","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223719]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.719785222Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.719+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*cixg2M inFlight=false count=2598","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223719]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.719805270Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.719+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*cixg2M count=2599","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223719]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Xp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.719825452Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.719+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=2600","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223719]},{"_index":"continuous-tests-pods-2023.11.09","_id":"X52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.719844194Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.719+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=2601","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223719]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.719851004Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.719+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*cixg2M count=2602","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223719]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.720007753Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.719+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*cixg2M inFlight=true count=2603","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223720]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Yp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.720100594Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.719+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*cixg2M count=2604","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223720]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Y52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.720128214Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.719+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=2605","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223720]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.720142442Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.720+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2606","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223720]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.720154178Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.720+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2607","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223720]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Zp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.720462285Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.720+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*cixg2M count=2608","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223720]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Z52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.721129996Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.720+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=2609","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223721]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.721309028Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.721+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*4rhgMa inflight=0 count=2610","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223721]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.733137964Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.732+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2611","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223733]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ap2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.733255914Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.732+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2612","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223733]},{"_index":"continuous-tests-pods-2023.11.09","_id":"a52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.734035845Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.733+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=2613","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223734]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.734112897Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.733+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/2M/zb2rhWniYEPPCn5GW9YTuXVtRAspDQtmUzzDHHMUyF8cixg2M + cid=zb2*cixg2M count=2614","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223734]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.734612641Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.734+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=967940 cid=zb2*cixg2M count=2615","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223734]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.739774479Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.739+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2616","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223739]},{"_index":"continuous-tests-pods-2023.11.09","_id":"b52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.739820996Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.739+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*cixg2M count=2617","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223739]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.739854534Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.739+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=20345 count=2618","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223739]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.739870724Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.739+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=2619","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223739]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.739941379Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.739+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*cixg2M queue=0 count=2620","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223739]},{"_index":"continuous-tests-pods-2023.11.09","_id":"c52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.739951942Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.739+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2621","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223739]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.740086258Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.739+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2622","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223740]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.740094827Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.740+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=2623","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223740]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.740307530Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.740+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*cixg2M count=2624","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223740]},{"_index":"continuous-tests-pods-2023.11.09","_id":"d52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.740495464Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.740+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*cixg2M inflight=1 count=2625","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223740]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.740573781Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.740+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*cixg2M count=2626","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223740]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.740775182Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.740+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=13 cid=zb2*cixg2M bytes=65472 blockOffset=0 count=2627","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223740]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ep2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.740812216Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.740+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=2628","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223740]},{"_index":"continuous-tests-pods-2023.11.09","_id":"e52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.741116009Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.740+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=2629","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223741]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.741144713Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.741+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*GBeWSm count=2630","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223741]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.741334171Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.741+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*GBeWSm count=2631","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223741]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.741351250Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.741+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*GBeWSm peers=1 count=2632","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223741]},{"_index":"continuous-tests-pods-2023.11.09","_id":"f52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.741447996Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.741+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*GBeWSm inFlight=false count=2633","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223741]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.741462927Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.741+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*GBeWSm count=2634","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223741]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.741597885Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.741+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=2635","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223741]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.741607482Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.741+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=2636","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223741]},{"_index":"continuous-tests-pods-2023.11.09","_id":"g52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.741679287Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.741+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*GBeWSm count=2637","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223741]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.741727332Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.741+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*GBeWSm inFlight=true count=2638","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223741]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.741747629Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.741+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*GBeWSm count=2639","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223741]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.741778106Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.741+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=2640","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223741]},{"_index":"continuous-tests-pods-2023.11.09","_id":"h52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.741833984Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.741+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2641","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223741]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.741943175Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.741+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2642","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223741]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.742116893Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.742+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*GBeWSm count=2643","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223742]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ip2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.743862357Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.743+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=2644","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223743]},{"_index":"continuous-tests-pods-2023.11.09","_id":"i52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.743907743Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.743+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*cixg2M inflight=0 count=2645","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223743]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.748708044Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.748+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2646","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223748]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.748754593Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.748+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2647","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223748]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.749358426Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.749+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=2648","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223749]},{"_index":"continuous-tests-pods-2023.11.09","_id":"j52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.749508624Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.749+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/Sm/zb2rhn76Ls8hoAhJaKoqgjgFUHppDLUxVFs4KJNDuGmGBeWSm + cid=zb2*GBeWSm count=2649","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223749]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.750001716Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.749+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=1033412 cid=zb2*GBeWSm count=2650","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223750]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.755217344Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.754+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2651","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223755]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.755247012Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.755+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*GBeWSm count=2652","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223755]},{"_index":"continuous-tests-pods-2023.11.09","_id":"k52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.755254058Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.755+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=13730 count=2653","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223755]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.755360119Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.755+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=2654","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223755]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.755384741Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.755+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*GBeWSm queue=0 count=2655","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223755]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.755430509Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.755+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2656","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223755]},{"_index":"continuous-tests-pods-2023.11.09","_id":"l52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.755453914Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.755+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2657","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223755]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.755460425Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.755+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=2658","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223755]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.755729911Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.755+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*GBeWSm count=2659","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223755]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.755852516Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.755+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*GBeWSm inflight=1 count=2660","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223755]},{"_index":"continuous-tests-pods-2023.11.09","_id":"m52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.755861871Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.755+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*GBeWSm count=2661","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223755]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.756163407Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.755+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=14 cid=zb2*GBeWSm bytes=65472 blockOffset=0 count=2662","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223756]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.756184742Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.756+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=2663","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223756]},{"_index":"continuous-tests-pods-2023.11.09","_id":"np2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.756368283Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.756+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=2664","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223756]},{"_index":"continuous-tests-pods-2023.11.09","_id":"n52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.756377609Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.756+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*Mkmv8p count=2665","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223756]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.756597707Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.756+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*Mkmv8p count=2666","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223756]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.756606611Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.756+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*Mkmv8p peers=1 count=2667","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223756]},{"_index":"continuous-tests-pods-2023.11.09","_id":"op2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.756656632Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.756+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*Mkmv8p inFlight=false count=2668","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223756]},{"_index":"continuous-tests-pods-2023.11.09","_id":"o52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.756867429Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.756+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*Mkmv8p count=2669","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223756]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.756875380Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.756+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=2670","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223756]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.756880661Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.756+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=2671","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223756]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.756922649Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.756+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*Mkmv8p count=2672","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223756]},{"_index":"continuous-tests-pods-2023.11.09","_id":"p52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.756929904Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.756+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*Mkmv8p inFlight=true count=2673","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223756]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.757053107Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.756+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*Mkmv8p count=2674","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223757]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.757061145Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.756+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=2675","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223757]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.757092787Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.757+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2676","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223757]},{"_index":"continuous-tests-pods-2023.11.09","_id":"q52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.757145242Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.757+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2677","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223757]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.757329326Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.757+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*Mkmv8p count=2678","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223757]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.757966575Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.757+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=2679","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223757]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.757977118Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.757+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*GBeWSm inflight=0 count=2680","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223757]},{"_index":"continuous-tests-pods-2023.11.09","_id":"r52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.770678943Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.770+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2681","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223770]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.770717611Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.770+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2682","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223770]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.771640421Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.771+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=2683","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223771]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.771678455Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.771+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/8p/zb2rhh2SpiteVHwTjswCiT723pZHWoRKPQ9rVYvHdDQMkmv8p + cid=zb2*Mkmv8p count=2684","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223771]},{"_index":"continuous-tests-pods-2023.11.09","_id":"s52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.772426991Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.772+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=1098884 cid=zb2*Mkmv8p count=2685","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223772]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.779062720Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.778+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2686","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223779]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.779097759Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.778+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*Mkmv8p count=2687","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223779]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.779114789Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.778+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=22401 count=2688","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223779]},{"_index":"continuous-tests-pods-2023.11.09","_id":"t52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.779120236Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.779+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=2689","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223779]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.779136892Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.779+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*Mkmv8p queue=0 count=2690","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223779]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.779189128Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.779+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2691","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223779]},{"_index":"continuous-tests-pods-2023.11.09","_id":"up2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.779316609Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.779+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2692","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223779]},{"_index":"continuous-tests-pods-2023.11.09","_id":"u52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.779327560Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.779+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=2693","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223779]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.779502379Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.779+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*Mkmv8p count=2694","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223779]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.779760737Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.779+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*Mkmv8p inflight=1 count=2695","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223779]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.779830713Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.779+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*Mkmv8p count=2696","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223779]},{"_index":"continuous-tests-pods-2023.11.09","_id":"v52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.780053565Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.779+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=15 cid=zb2*Mkmv8p bytes=65472 blockOffset=0 count=2697","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223780]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.780075939Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.779+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=2698","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223780]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.780328595Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.780+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=2699","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223780]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.780423695Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.780+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*JwTESK count=2700","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223780]},{"_index":"continuous-tests-pods-2023.11.09","_id":"w52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.780610978Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.780+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*JwTESK count=2701","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223780]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.780643606Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.780+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*JwTESK peers=1 count=2702","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223780]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.780856730Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.780+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*JwTESK inFlight=false count=2703","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223780]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.780873804Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.780+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*JwTESK count=2704","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223780]},{"_index":"continuous-tests-pods-2023.11.09","_id":"x52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.780951162Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.780+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=2705","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223780]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.780959676Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.780+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=2706","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223780]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.781061173Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.780+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*JwTESK count=2707","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223781]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.781077762Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.781+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*JwTESK inFlight=true count=2708","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223781]},{"_index":"continuous-tests-pods-2023.11.09","_id":"y52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.781195428Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.781+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*JwTESK count=2709","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223781]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zJ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.781209954Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.781+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=2710","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223781]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zZ2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.781285018Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.781+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2711","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223781]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zp2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.781412433Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.781+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2712","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223781]},{"_index":"continuous-tests-pods-2023.11.09","_id":"z52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.781586081Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.781+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*JwTESK count=2713","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223781]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0J2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.782176709Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.781+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=2714","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223782]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0Z2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.782191758Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.782+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*Mkmv8p inflight=0 count=2715","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223782]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0p2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.795436668Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.795+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2716","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223795]},{"_index":"continuous-tests-pods-2023.11.09","_id":"052EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.795475041Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.795+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2717","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223795]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1J2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.795970561Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.795+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=2718","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223795]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1Z2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.796131741Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.796+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/SK/zb2rhYwo2irKAfw235FYUa6naymGw7a8bW286cGfFAVJwTESK + cid=zb2*JwTESK count=2719","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223796]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1p2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.796535414Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.796+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=1164356 cid=zb2*JwTESK count=2720","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223796]},{"_index":"continuous-tests-pods-2023.11.09","_id":"152EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.802580334Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.802+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2721","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223802]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2J2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.802608903Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.802+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*JwTESK count=2722","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223802]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2Z2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.802613718Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.802+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=21638 count=2723","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223802]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2p2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.802635672Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.802+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=2724","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223802]},{"_index":"continuous-tests-pods-2023.11.09","_id":"252EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.802639959Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.802+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*JwTESK queue=0 count=2725","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223802]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3J2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.802706909Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.802+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2726","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223802]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3Z2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.802729614Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.802+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2727","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223802]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3p2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.802818980Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.802+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=2728","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223802]},{"_index":"continuous-tests-pods-2023.11.09","_id":"352EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.802916619Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.802+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*JwTESK count=2729","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223802]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4J2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.803170987Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.802+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*JwTESK inflight=1 count=2730","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223803]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4Z2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.803196634Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.803+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*JwTESK count=2731","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223803]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4p2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.803332249Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.803+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=16 cid=zb2*JwTESK bytes=65472 blockOffset=0 count=2732","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223803]},{"_index":"continuous-tests-pods-2023.11.09","_id":"452EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.803431655Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.803+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=2733","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223803]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5J2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.803637099Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.803+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=2734","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223803]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5Z2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.803644400Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.803+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*TyyAZA count=2735","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223803]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5p2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.803849964Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.803+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*TyyAZA count=2736","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223803]},{"_index":"continuous-tests-pods-2023.11.09","_id":"552EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.803856043Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.803+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*TyyAZA peers=1 count=2737","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223803]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6J2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.803997966Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.803+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*TyyAZA inFlight=false count=2738","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223803]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6Z2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.804017879Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.803+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*TyyAZA count=2739","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223804]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6p2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.804115456Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.803+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=2740","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223804]},{"_index":"continuous-tests-pods-2023.11.09","_id":"652EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.804122359Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.804+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=2741","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223804]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7J2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.804199303Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.804+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*TyyAZA count=2742","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223804]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7Z2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.804204444Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.804+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*TyyAZA inFlight=true count=2743","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223804]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7p2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.804276749Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.804+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*TyyAZA count=2744","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223804]},{"_index":"continuous-tests-pods-2023.11.09","_id":"752EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.804374179Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.804+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=2745","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223804]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8J2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.804396941Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.804+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2746","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223804]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8Z2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.804457297Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.804+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2747","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223804]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8p2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.804586451Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.804+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*TyyAZA count=2748","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223804]},{"_index":"continuous-tests-pods-2023.11.09","_id":"852EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.805199281Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.805+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=2749","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223805]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9J2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.805216160Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.805+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*JwTESK inflight=0 count=2750","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223805]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9Z2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.817484248Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.817+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2751","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223817]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9p2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.817522746Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.817+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2752","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223817]},{"_index":"continuous-tests-pods-2023.11.09","_id":"952EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.818710110Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.818+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=2753","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223818]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-J2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.818740322Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.818+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/ZA/zb2rhijgoj5MoU3peCRMhbsR8SfKyAHajhu8Q8QnqWiTyyAZA + cid=zb2*TyyAZA count=2754","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223818]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-Z2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.819258772Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.819+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=1229828 cid=zb2*TyyAZA count=2755","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223819]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-p2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.824727683Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.824+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2756","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223824]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.824759566Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.824+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*TyyAZA count=2757","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223824]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_J2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.824854861Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.824+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=20765 count=2758","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223824]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_Z2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.824882108Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.824+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=2759","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223824]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_p2EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.825051380Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.824+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*TyyAZA queue=0 count=2760","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223825]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_52EtosBOvK4VyD1lKc-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.825064890Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.824+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2761","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223825]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AJ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.825234348Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.825+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2762","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223825]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AZ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.825245192Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.825+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=2763","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223825]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ap2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.825499297Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.825+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*TyyAZA count=2764","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223825]},{"_index":"continuous-tests-pods-2023.11.09","_id":"A52EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.825754173Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.825+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*TyyAZA inflight=1 count=2765","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223825]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BJ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.826017404Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.825+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*TyyAZA count=2766","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223826]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BZ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.826174635Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.826+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=17 cid=zb2*TyyAZA bytes=65472 blockOffset=0 count=2767","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223826]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Bp2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.826310568Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.826+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=2768","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223826]},{"_index":"continuous-tests-pods-2023.11.09","_id":"B52EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.826728126Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.826+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=2769","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223826]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CJ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.826856453Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.826+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*N3JeC4 count=2770","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223826]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CZ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.827135343Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.827+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*N3JeC4 count=2771","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223827]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Cp2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.827154586Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.827+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*N3JeC4 peers=1 count=2772","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223827]},{"_index":"continuous-tests-pods-2023.11.09","_id":"C52EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.827343471Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.827+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*N3JeC4 inFlight=false count=2773","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223827]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DJ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.827357205Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.827+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*N3JeC4 count=2774","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223827]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DZ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.827453788Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.827+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=2775","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223827]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Dp2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.827534044Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.827+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=2776","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223827]},{"_index":"continuous-tests-pods-2023.11.09","_id":"D52EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.827651498Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.827+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*N3JeC4 count=2777","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223827]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EJ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.827831583Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.827+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*N3JeC4 inFlight=true count=2778","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223827]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EZ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.827848485Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.827+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*N3JeC4 count=2779","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223827]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ep2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.827934194Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.827+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=2780","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223827]},{"_index":"continuous-tests-pods-2023.11.09","_id":"E52EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.828015854Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.827+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2781","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223828]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FJ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.828119026Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.827+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2782","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223828]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FZ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.828250439Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.828+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*N3JeC4 count=2783","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223828]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Fp2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.828842932Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.828+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=2784","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223828]},{"_index":"continuous-tests-pods-2023.11.09","_id":"F52EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.828877053Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.828+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*TyyAZA inflight=0 count=2785","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223828]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GJ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.840762176Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.840+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2786","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223840]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GZ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.840803644Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.840+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2787","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223840]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Gp2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.841678645Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.841+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=2788","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223841]},{"_index":"continuous-tests-pods-2023.11.09","_id":"G52EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.841800384Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.841+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/C4/zb2rhoArFR2rg16RQLV1MfjRwSa5KWTBWW2GJhPPHqSN3JeC4 + cid=zb2*N3JeC4 count=2789","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223841]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HJ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.842233109Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.842+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=1295300 cid=zb2*N3JeC4 count=2790","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223842]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HZ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.848615568Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.848+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2791","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223848]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Hp2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.848646922Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.848+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*N3JeC4 count=2792","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223848]},{"_index":"continuous-tests-pods-2023.11.09","_id":"H52EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.848652556Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.848+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=21395 count=2793","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223848]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IJ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.848794081Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.848+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=2794","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223848]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IZ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.848952946Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.848+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*N3JeC4 queue=0 count=2795","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223848]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ip2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.849076735Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.848+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2796","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223849]},{"_index":"continuous-tests-pods-2023.11.09","_id":"I52EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.849083700Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.849+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2797","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223849]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JJ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.849207196Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.849+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=2798","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223849]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JZ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.849416867Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.849+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*N3JeC4 count=2799","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223849]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Jp2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.849873752Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.849+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*N3JeC4 inflight=1 count=2800","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223849]},{"_index":"continuous-tests-pods-2023.11.09","_id":"J52EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.849887190Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.849+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*N3JeC4 count=2801","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223849]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KJ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.850046579Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.849+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=18 cid=zb2*N3JeC4 bytes=65472 blockOffset=0 count=2802","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223850]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KZ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.850263452Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.850+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=2803","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223850]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Kp2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.850516686Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.850+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=2804","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223850]},{"_index":"continuous-tests-pods-2023.11.09","_id":"K52EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.850527286Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.850+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*s2fUdt count=2805","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223850]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LJ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.850794207Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.850+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*s2fUdt count=2806","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223850]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LZ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.850819385Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.850+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*s2fUdt peers=1 count=2807","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223850]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Lp2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.850831133Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.850+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*s2fUdt inFlight=false count=2808","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223850]},{"_index":"continuous-tests-pods-2023.11.09","_id":"L52EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.851101645Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.850+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*s2fUdt count=2809","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223851]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MJ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.851191317Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.851+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=2810","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223851]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MZ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.851201173Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.851+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=2811","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223851]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Mp2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.851293040Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.851+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*s2fUdt count=2812","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223851]},{"_index":"continuous-tests-pods-2023.11.09","_id":"M52EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.851370053Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.851+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*s2fUdt inFlight=true count=2813","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223851]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NJ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.851467624Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.851+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*s2fUdt count=2814","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223851]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NZ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.851539840Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.851+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=2815","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223851]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Np2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.851590428Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.851+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2816","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223851]},{"_index":"continuous-tests-pods-2023.11.09","_id":"N52EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.851597405Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.851+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2817","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223851]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OJ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.851767360Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.851+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*s2fUdt count=2818","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223851]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OZ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.852101461Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.852+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=2819","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223852]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Op2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.852301505Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.852+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*N3JeC4 inflight=0 count=2820","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223852]},{"_index":"continuous-tests-pods-2023.11.09","_id":"O52EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.863489838Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.863+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2821","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223863]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PJ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.863523331Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.863+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2822","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223863]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PZ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.864229812Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.864+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=2823","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223864]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Pp2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.864494701Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.864+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/dt/zb2rhccErVnKUdCqZGbxi4DQBAsJ88qkb9XmdknXrF2s2fUdt + cid=zb2*s2fUdt count=2824","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223864]},{"_index":"continuous-tests-pods-2023.11.09","_id":"P52EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.865001924Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.864+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=1360772 cid=zb2*s2fUdt count=2825","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223865]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QJ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.872351112Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.872+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2826","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223872]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QZ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.872396307Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.872+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*s2fUdt count=2827","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223872]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Qp2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.872402120Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.872+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=21385 count=2828","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223872]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Q52EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.872480693Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.872+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=2829","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223872]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RJ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.872508787Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.872+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*s2fUdt queue=0 count=2830","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223872]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RZ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.872514868Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.872+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2831","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223872]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Rp2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.872582427Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.872+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2832","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223872]},{"_index":"continuous-tests-pods-2023.11.09","_id":"R52EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.872608435Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.872+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=2833","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223872]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SJ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.873015287Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.872+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*s2fUdt count=2834","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223873]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SZ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.873156960Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.873+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*s2fUdt inflight=1 count=2835","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223873]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Sp2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.873197441Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.873+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*s2fUdt count=2836","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223873]},{"_index":"continuous-tests-pods-2023.11.09","_id":"S52EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.873483408Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.873+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=19 cid=zb2*s2fUdt bytes=65472 blockOffset=0 count=2837","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223873]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TJ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.873494280Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.873+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=2838","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223873]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TZ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.873664300Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.873+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=2839","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223873]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Tp2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.873672425Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.873+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*ujzude count=2840","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223873]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-Z2EtosBOvK4VyD1lauU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.874988211Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.874+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2852","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223874]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-p2EtosBOvK4VyD1lauU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.874991946Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.874+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*ujzude count=2853","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223874]},{"_index":"continuous-tests-pods-2023.11.09","_id":"T52EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.874828767Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.873+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*ujzude count=2841","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223874]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UJ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.874942626Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.873+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*ujzude peers=1 count=2842","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223874]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UZ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.874951655Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.873+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*ujzude inFlight=false count=2843","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223874]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Up2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.874956038Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.874+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*ujzude count=2844","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223874]},{"_index":"continuous-tests-pods-2023.11.09","_id":"U52EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.874960075Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.874+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=2845","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223874]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VJ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.874964732Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.874+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=2846","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223874]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VZ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.874968680Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.874+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*ujzude count=2847","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223874]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Vp2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.874972445Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.874+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*ujzude inFlight=true count=2848","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223874]},{"_index":"continuous-tests-pods-2023.11.09","_id":"V52EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.874976322Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.874+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*ujzude count=2849","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223874]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WJ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.874980852Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.874+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=2850","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223874]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WZ2EtosBOvK4VyD1lKg-","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.874984579Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.874+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2851","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223874]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-52EtosBOvK4VyD1lauU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.875494467Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.875+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=2854","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223875]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_J2EtosBOvK4VyD1lauU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.875621896Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.875+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*s2fUdt inflight=0 count=2855","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223875]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_Z2EtosBOvK4VyD1lauU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.881041139Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.880+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2856","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223881]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_p2EtosBOvK4VyD1lauU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.881108559Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.880+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2857","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223881]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_52EtosBOvK4VyD1lauU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.881806645Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.881+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=2858","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223881]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.882064241Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.881+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/de/zb2rhc23F387w61Rd91HyuLhb3HKfYaKLN8JSyuFdPDujzude + cid=zb2*ujzude count=2859","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223882]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.882498922Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.882+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=1426244 cid=zb2*ujzude count=2860","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223882]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ap2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.888777636Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.888+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2861","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223888]},{"_index":"continuous-tests-pods-2023.11.09","_id":"A52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.888806301Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.888+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*ujzude count=2862","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223888]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.888811599Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.888+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=14638 count=2863","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223888]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.888815941Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.888+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=2864","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223888]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Bp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.888819917Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.888+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*ujzude queue=0 count=2865","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223888]},{"_index":"continuous-tests-pods-2023.11.09","_id":"B52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.888877471Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.888+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2866","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223888]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.888931931Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.888+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2867","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223888]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.888938292Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.888+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=2868","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223888]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Cp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.889052357Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.888+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*ujzude count=2869","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223889]},{"_index":"continuous-tests-pods-2023.11.09","_id":"C52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.889309262Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.889+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*ujzude inflight=1 count=2870","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223889]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.889328348Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.889+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*ujzude count=2871","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223889]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.889484736Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.889+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=20 cid=zb2*ujzude bytes=65472 blockOffset=0 count=2872","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223889]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Dp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.889557560Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.889+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=2873","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223889]},{"_index":"continuous-tests-pods-2023.11.09","_id":"D52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.889773824Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.889+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=2874","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223889]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.889786954Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.889+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*vEVWpc count=2875","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223889]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.889963695Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.889+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*vEVWpc count=2876","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223889]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ep2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.889981704Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.889+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*vEVWpc peers=1 count=2877","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223889]},{"_index":"continuous-tests-pods-2023.11.09","_id":"E52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.889993514Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.889+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*vEVWpc inFlight=false count=2878","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223889]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.890122867Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.889+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*vEVWpc count=2879","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223890]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.890150615Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.890+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=2880","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223890]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Fp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.890156141Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.890+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=2881","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223890]},{"_index":"continuous-tests-pods-2023.11.09","_id":"F52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.890766551Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.890+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*vEVWpc count=2882","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223890]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.890784086Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.890+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*vEVWpc inFlight=true count=2883","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223890]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.890789251Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.890+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*vEVWpc count=2884","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223890]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Gp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.890794265Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.890+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=2885","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223890]},{"_index":"continuous-tests-pods-2023.11.09","_id":"G52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.890807155Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.890+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2886","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223890]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.890811815Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.890+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2887","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223890]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.890816624Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.890+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*vEVWpc count=2888","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223890]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Hp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.891482435Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.891+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=2889","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223891]},{"_index":"continuous-tests-pods-2023.11.09","_id":"H52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.891600345Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.891+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*ujzude inflight=0 count=2890","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223891]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.902984621Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.902+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2891","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223902]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.903028077Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.902+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2892","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223903]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ip2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.903781016Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.903+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=2893","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223903]},{"_index":"continuous-tests-pods-2023.11.09","_id":"I52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.904111280Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.903+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/pc/zb2rhogz99EaNhjnK36WcrfBgJh1xHqAjp3WmwkBnHNvEVWpc + cid=zb2*vEVWpc count=2894","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223904]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.905612202Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.904+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=1491716 cid=zb2*vEVWpc count=2895","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223905]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.910777779Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.910+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2896","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223910]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Jp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.910815498Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.910+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*vEVWpc count=2897","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223910]},{"_index":"continuous-tests-pods-2023.11.09","_id":"J52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.910829768Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.910+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=20496 count=2898","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223910]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.910834390Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.910+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=2899","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223910]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.910838807Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.910+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*vEVWpc queue=0 count=2900","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223910]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Kp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.910843436Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.910+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2901","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223910]},{"_index":"continuous-tests-pods-2023.11.09","_id":"K52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.910870107Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.910+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2902","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223910]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.910906517Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.910+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=2903","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223910]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Lp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.911298898Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.911+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*vEVWpc inflight=1 count=2905","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223911]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.911465251Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.911+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=2908","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223911]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.911853658Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.911+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*DzvffK count=2911","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223911]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.911019029Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.910+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*vEVWpc count=2904","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223911]},{"_index":"continuous-tests-pods-2023.11.09","_id":"L52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.911325551Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.911+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*vEVWpc count=2906","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223911]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.911448573Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.911+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=21 cid=zb2*vEVWpc bytes=65472 blockOffset=0 count=2907","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223911]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Mp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.911687020Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.911+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=2909","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223911]},{"_index":"continuous-tests-pods-2023.11.09","_id":"M52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.911697482Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.911+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*DzvffK count=2910","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223911]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.911863159Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.911+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*DzvffK peers=1 count=2912","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223911]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Np2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.911899089Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.911+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*DzvffK inFlight=false count=2913","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223911]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.912210083Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.911+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=2915","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.912218404Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.912+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=2916","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Op2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.912223324Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.912+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*DzvffK count=2917","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"O52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.912228106Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.912+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*DzvffK inFlight=true count=2918","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.912362710Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.912+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*DzvffK count=2919","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Pp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.912374128Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.912+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2921","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"P52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.912489278Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.912+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2922","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.912603188Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.912+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*DzvffK count=2923","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"N52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.912092549Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.911+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*DzvffK count=2914","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.912369822Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.912+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=2920","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.913253296Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.913+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=2924","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223913]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Qp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.913276760Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.913+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*vEVWpc inflight=0 count=2925","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223913]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Q52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.927250022Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.926+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2926","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223927]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.927474363Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.927+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2927","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223927]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.928417404Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.928+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=2928","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223928]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Rp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.928465836Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.928+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/fK/zb2rhZwB8GoCpeXQ8DwHXTN7uBfqun3GCmxUXQZc6skDzvffK + cid=zb2*DzvffK count=2929","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223928]},{"_index":"continuous-tests-pods-2023.11.09","_id":"R52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.929176970Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.928+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=1557188 cid=zb2*DzvffK count=2930","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223929]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.934660024Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.934+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*DzvffK queue=0 count=2935","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223934]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.934695421Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.934+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2936","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223934]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Tp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.934768804Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.934+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2937","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223934]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.934503774Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.934+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2931","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223934]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.934533784Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.934+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*DzvffK count=2932","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223934]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Sp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.934552385Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.934+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=22584 count=2933","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223934]},{"_index":"continuous-tests-pods-2023.11.09","_id":"S52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.934557060Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.934+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=2934","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223934]},{"_index":"continuous-tests-pods-2023.11.09","_id":"T52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.934775125Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.934+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=2938","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223934]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.934968794Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.934+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*DzvffK count=2939","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223934]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.935778997Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.935+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=2944","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223935]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Vp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.935882037Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.935+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*EPSXHb count=2945","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223935]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.935279778Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.935+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*DzvffK inflight=1 count=2940","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223935]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Up2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.935289651Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.935+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*DzvffK count=2941","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223935]},{"_index":"continuous-tests-pods-2023.11.09","_id":"U52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.935567688Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.935+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=22 cid=zb2*DzvffK bytes=65472 blockOffset=0 count=2942","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223935]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.935582166Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.935+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=2943","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223935]},{"_index":"continuous-tests-pods-2023.11.09","_id":"V52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.936024420Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.935+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*EPSXHb count=2946","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223936]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.936031978Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.935+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*EPSXHb peers=1 count=2947","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223936]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.936356456Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.936+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*EPSXHb inFlight=false count=2948","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223936]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.936379030Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.936+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*EPSXHb count=2952","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223936]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Xp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.936465549Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.936+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*EPSXHb inFlight=true count=2953","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223936]},{"_index":"continuous-tests-pods-2023.11.09","_id":"X52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.936471673Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.936+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*EPSXHb count=2954","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223936]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.936523196Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.936+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=2955","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223936]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.936554995Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.936+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2956","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223936]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Yp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.936832367Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.936+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2957","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223936]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Y52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.936857381Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.936+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*EPSXHb count=2958","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223936]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Wp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.936366329Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.936+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*EPSXHb count=2949","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223936]},{"_index":"continuous-tests-pods-2023.11.09","_id":"W52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.936370398Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.936+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=2950","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223936]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.936375127Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.936+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=2951","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223936]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.937541523Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.937+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=2959","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223937]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.937648961Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.937+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*DzvffK inflight=0 count=2960","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223937]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Zp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.951411677Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.951+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2961","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223951]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Z52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.951489095Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.951+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2962","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223951]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.952088050Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.951+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=2963","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223952]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.952208208Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.952+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/Hb/zb2rhoEw5gKGsFH5HVhzn3zK5N1haqhDKMqSagc8AgqEPSXHb + cid=zb2*EPSXHb count=2964","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223952]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ap2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.952635664Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.952+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=1622660 cid=zb2*EPSXHb count=2965","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223952]},{"_index":"continuous-tests-pods-2023.11.09","_id":"a52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.957830060Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.957+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2966","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223957]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.957889741Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.957+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*EPSXHb count=2967","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223957]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.957927012Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.957+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=21790 count=2968","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223957]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.958163654Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.957+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=2971","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223958]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.958172143Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.958+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=2972","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223958]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.958176724Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.958+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=2973","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223958]},{"_index":"continuous-tests-pods-2023.11.09","_id":"c52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.958395096Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.958+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*EPSXHb count=2974","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223958]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.958643155Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.958+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*EPSXHb inflight=1 count=2975","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223958]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.958650061Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.958+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*EPSXHb count=2976","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223958]},{"_index":"continuous-tests-pods-2023.11.09","_id":"d52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.958979122Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.958+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=2978","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223958]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.958026533Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.957+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=2969","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223958]},{"_index":"continuous-tests-pods-2023.11.09","_id":"b52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.958041751Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.957+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*EPSXHb queue=0 count=2970","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223958]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.958933878Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.958+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=23 cid=zb2*EPSXHb bytes=65472 blockOffset=0 count=2977","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223958]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.959418871Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.959+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=2979","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223959]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.959437251Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.959+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*pbKCxZ count=2980","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223959]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ep2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.959693552Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.959+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*pbKCxZ count=2981","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223959]},{"_index":"continuous-tests-pods-2023.11.09","_id":"e52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.959711446Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.959+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*pbKCxZ peers=1 count=2982","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223959]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.959716457Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.959+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*pbKCxZ inFlight=false count=2983","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223959]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.959842372Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.959+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*pbKCxZ count=2984","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223959]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.959926504Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.959+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=2985","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223959]},{"_index":"continuous-tests-pods-2023.11.09","_id":"f52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.959945604Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.959+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=2986","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223959]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.960208088Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.959+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*pbKCxZ count=2987","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223960]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.960226831Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.960+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*pbKCxZ inFlight=true count=2988","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223960]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.960239184Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.960+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*pbKCxZ count=2989","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223960]},{"_index":"continuous-tests-pods-2023.11.09","_id":"g52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.960339830Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.960+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=2990","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223960]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.960496744Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.960+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=2991","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223960]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.960542064Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.960+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2992","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223960]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.960783291Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.960+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*pbKCxZ count=2993","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223960]},{"_index":"continuous-tests-pods-2023.11.09","_id":"h52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.961389650Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.961+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=2994","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223961]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.961445428Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.961+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*EPSXHb inflight=0 count=2995","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223961]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.974073395Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.973+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=2996","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223974]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ip2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.974120602Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.973+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=2997","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223974]},{"_index":"continuous-tests-pods-2023.11.09","_id":"i52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.974796141Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.974+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=2998","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223974]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.975040957Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.974+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/xZ/zb2rhaHM3CGXx7BMbhKhA9xpi1murJSkhFJfuEHctkqpbKCxZ + cid=zb2*pbKCxZ count=2999","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223975]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.975423764Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.975+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=1688132 cid=zb2*pbKCxZ count=3000","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223975]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.981164967Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.980+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3001","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223981]},{"_index":"continuous-tests-pods-2023.11.09","_id":"j52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.981205376Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.980+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*pbKCxZ count=3002","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223981]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.981213176Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.981+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=21431 count=3003","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223981]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.981254801Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.981+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3004","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223981]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.981263623Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.981+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*pbKCxZ queue=0 count=3005","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223981]},{"_index":"continuous-tests-pods-2023.11.09","_id":"k52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.981322123Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.981+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3006","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223981]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.981334703Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.981+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3007","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223981]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.981343625Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.981+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3008","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223981]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.981670728Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.981+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*pbKCxZ count=3009","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223981]},{"_index":"continuous-tests-pods-2023.11.09","_id":"l52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.981786660Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.981+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*pbKCxZ inflight=1 count=3010","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223981]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.981827633Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.981+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*pbKCxZ count=3011","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223981]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.982107036Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.981+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=24 cid=zb2*pbKCxZ bytes=65472 blockOffset=0 count=3012","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223982]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.982141197Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.982+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3013","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223982]},{"_index":"continuous-tests-pods-2023.11.09","_id":"m52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.982528597Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.982+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3014","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223982]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.982548190Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.982+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*6DSvXG count=3015","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223982]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.982676607Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.982+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*6DSvXG count=3016","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223982]},{"_index":"continuous-tests-pods-2023.11.09","_id":"np2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.982690216Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.982+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*6DSvXG peers=1 count=3017","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223982]},{"_index":"continuous-tests-pods-2023.11.09","_id":"n52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.982924818Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.982+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*6DSvXG inFlight=false count=3018","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223982]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.982985118Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.982+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*6DSvXG count=3019","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223982]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.983017305Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.982+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3020","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223983]},{"_index":"continuous-tests-pods-2023.11.09","_id":"op2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.983023681Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.982+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3021","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223983]},{"_index":"continuous-tests-pods-2023.11.09","_id":"o52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.983040499Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.982+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*6DSvXG count=3022","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223983]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.983127185Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.983+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*6DSvXG inFlight=true count=3023","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223983]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.983203107Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.983+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*6DSvXG count=3024","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223983]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.983209577Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.983+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3025","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223983]},{"_index":"continuous-tests-pods-2023.11.09","_id":"p52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.983294629Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.983+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3026","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223983]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.983301242Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.983+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3027","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223983]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.983514719Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.983+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*6DSvXG count=3028","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223983]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.983789309Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.983+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3029","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223983]},{"_index":"continuous-tests-pods-2023.11.09","_id":"q52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.983975590Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.983+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*pbKCxZ inflight=0 count=3030","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223983]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.996944635Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.996+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3031","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223996]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.996988062Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.996+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3032","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223996]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:03.997879283Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:03.997+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3033","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574223997]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.005586852Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.005+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=1753604 cid=zb2*6DSvXG count=3035","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224005]},{"_index":"continuous-tests-pods-2023.11.09","_id":"r52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.005075312Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.004+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/XG/zb2rhbCFxfiBkeQ5c47NRK5B8jAR9SEtGeU7cJTysEv6DSvXG + cid=zb2*6DSvXG count=3034","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224005]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.010836733Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.010+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3036","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224010]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.010894037Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.010+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*6DSvXG count=3037","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224010]},{"_index":"continuous-tests-pods-2023.11.09","_id":"s52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.010911996Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.010+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=28033 count=3038","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224010]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.011009574Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.010+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3039","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224011]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.011021489Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.010+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*6DSvXG queue=0 count=3040","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224011]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.011097800Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.010+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3041","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224011]},{"_index":"continuous-tests-pods-2023.11.09","_id":"t52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.011106403Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.011+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3042","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224011]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.011212467Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.011+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3043","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224011]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.011350177Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.011+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*6DSvXG count=3044","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224011]},{"_index":"continuous-tests-pods-2023.11.09","_id":"up2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.011517613Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.011+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*6DSvXG inflight=1 count=3045","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224011]},{"_index":"continuous-tests-pods-2023.11.09","_id":"u52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.011612677Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.011+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*6DSvXG count=3046","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224011]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.011797717Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.011+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=25 cid=zb2*6DSvXG bytes=65472 blockOffset=0 count=3047","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224011]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.011812500Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.011+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3048","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224011]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.012110022Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.011+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3049","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224012]},{"_index":"continuous-tests-pods-2023.11.09","_id":"v52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.012143365Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.011+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*cMHCbe count=3050","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224012]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.012249445Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.012+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*cMHCbe count=3051","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224012]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.012259285Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.012+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*cMHCbe peers=1 count=3052","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224012]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.012367856Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.012+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*cMHCbe inFlight=false count=3053","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224012]},{"_index":"continuous-tests-pods-2023.11.09","_id":"w52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.012390100Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.012+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*cMHCbe count=3054","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224012]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.012451610Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.012+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3055","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224012]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.012459908Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.012+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3056","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224012]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.012464454Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.012+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*cMHCbe count=3057","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224012]},{"_index":"continuous-tests-pods-2023.11.09","_id":"x52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.012565117Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.012+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*cMHCbe inFlight=true count=3058","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224012]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.012696793Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.012+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*cMHCbe count=3059","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224012]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.012707915Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.012+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3060","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224012]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.012718826Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.012+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3061","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224012]},{"_index":"continuous-tests-pods-2023.11.09","_id":"y52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.012869671Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.012+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3062","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224012]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zJ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.012997449Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.012+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*cMHCbe count=3063","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224012]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zZ2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.013298257Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.013+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3064","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224013]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zp2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.013467044Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.013+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*6DSvXG inflight=0 count=3065","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224013]},{"_index":"continuous-tests-pods-2023.11.09","_id":"z52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.026688016Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.026+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3066","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224026]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0J2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.026814095Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.026+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3067","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224026]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0Z2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.027463899Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.027+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3068","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224027]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0p2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.027707606Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.027+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/be/zb2rhoZJqvMUJoG7p8gZ8fWKJqN5rvLVjiMxzUvZKsYcMHCbe + cid=zb2*cMHCbe count=3069","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224027]},{"_index":"continuous-tests-pods-2023.11.09","_id":"052EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.028070599Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.027+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=1819076 cid=zb2*cMHCbe count=3070","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224028]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1J2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.033670178Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.033+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3071","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224033]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1Z2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.033706102Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.033+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*cMHCbe count=3072","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224033]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1p2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.033711438Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.033+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=21418 count=3073","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224033]},{"_index":"continuous-tests-pods-2023.11.09","_id":"152EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.033718804Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.033+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3074","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224033]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2J2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.033788661Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.033+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*cMHCbe queue=0 count=3075","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224033]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2Z2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.033906373Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.033+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3076","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224033]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2p2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.033911958Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.033+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3077","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224033]},{"_index":"continuous-tests-pods-2023.11.09","_id":"252EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.033945382Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.033+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3078","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224033]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3J2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.034143645Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.034+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*cMHCbe count=3079","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224034]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3Z2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.034480602Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.034+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*cMHCbe inflight=1 count=3080","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224034]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3p2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.034501244Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.034+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*cMHCbe count=3081","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224034]},{"_index":"continuous-tests-pods-2023.11.09","_id":"352EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.034710809Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.034+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=26 cid=zb2*cMHCbe bytes=65472 blockOffset=0 count=3082","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224034]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4J2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.034793106Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.034+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3083","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224034]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4Z2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.034998143Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.034+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3084","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224034]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4p2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.035004924Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.034+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*fi79SB count=3085","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224035]},{"_index":"continuous-tests-pods-2023.11.09","_id":"452EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.035156178Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.035+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*fi79SB count=3086","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224035]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5J2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.035255796Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.035+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*fi79SB peers=1 count=3087","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224035]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5Z2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.035265389Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.035+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*fi79SB inFlight=false count=3088","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224035]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5p2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.035421410Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.035+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*fi79SB count=3089","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224035]},{"_index":"continuous-tests-pods-2023.11.09","_id":"652EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.035646379Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.035+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*fi79SB count=3094","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224035]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7Z2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.035777673Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.035+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3096","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224035]},{"_index":"continuous-tests-pods-2023.11.09","_id":"552EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.035427680Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.035+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3090","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224035]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6J2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.035439432Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.035+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3091","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224035]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6Z2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.035565084Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.035+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*fi79SB count=3092","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224035]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6p2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.035586639Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.035+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*fi79SB inFlight=true count=3093","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224035]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7J2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.035654178Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.035+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3095","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224035]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7p2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.035795398Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.035+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3097","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224035]},{"_index":"continuous-tests-pods-2023.11.09","_id":"752EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.035949684Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.035+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*fi79SB count=3098","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224035]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8Z2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.036735978Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.036+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*cMHCbe inflight=0 count=3100","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224036]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8J2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.036677644Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.036+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3099","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224036]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8p2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.048635473Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.048+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3101","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224048]},{"_index":"continuous-tests-pods-2023.11.09","_id":"852EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.048700133Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.048+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3102","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224048]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9J2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.049409188Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.049+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3103","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224049]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9Z2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.049458216Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.049+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/SB/zb2rhmShDz3PKbxLiDZbmf7PdtTUqoDGvZKQPvZMyURfi79SB + cid=zb2*fi79SB count=3104","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224049]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9p2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.050058774Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.049+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=1884548 cid=zb2*fi79SB count=3105","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224050]},{"_index":"continuous-tests-pods-2023.11.09","_id":"952EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.055273890Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.055+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3106","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224055]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-J2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.055309527Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.055+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*fi79SB count=3107","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224055]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-Z2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.055323165Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.055+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=19911 count=3108","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224055]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-p2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.055400715Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.055+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3109","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224055]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.055511117Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.055+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*fi79SB queue=0 count=3110","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224055]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_J2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.055687928Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.055+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3111","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224055]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_Z2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.055699456Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.055+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3112","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224055]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_p2EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.055807689Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.055+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3113","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224055]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_52EtosBOvK4VyD1layU","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.056043655Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.055+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*fi79SB count=3114","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224056]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.056293463Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.056+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*fi79SB inflight=1 count=3115","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224056]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.056476380Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.056+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*fi79SB count=3116","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224056]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ap2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.056704760Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.056+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=27 cid=zb2*fi79SB bytes=65472 blockOffset=0 count=3117","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224056]},{"_index":"continuous-tests-pods-2023.11.09","_id":"A52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.056817157Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.056+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3118","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224056]},{"_index":"continuous-tests-pods-2023.11.09","_id":"B52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.057570010Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.057+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*MhiATJ peers=1 count=3122","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.057175834Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.056+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3119","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.057190713Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.057+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*MhiATJ count=3120","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Bp2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.057466983Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.057+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*MhiATJ count=3121","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.057820593Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.057+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*MhiATJ inFlight=false count=3123","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.057942625Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.057+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*MhiATJ count=3124","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Cp2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.057959326Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.057+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3125","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"C52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.057963949Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.057+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3126","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ep2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.058996516Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.058+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*MhiATJ count=3133","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224058]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.058055941Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.057+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*MhiATJ count=3127","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224058]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.058208799Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.057+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*MhiATJ inFlight=true count=3128","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224058]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Dp2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.058409379Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.058+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*MhiATJ count=3129","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224058]},{"_index":"continuous-tests-pods-2023.11.09","_id":"D52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.058541887Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.058+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3130","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224058]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.058637603Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.058+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3131","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224058]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.058732022Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.058+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3132","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224058]},{"_index":"continuous-tests-pods-2023.11.09","_id":"E52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.059259111Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.059+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3134","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224059]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.059447111Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.059+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*fi79SB inflight=0 count=3135","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224059]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.076530070Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.076+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3136","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224076]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Fp2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.076586289Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.076+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3137","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224076]},{"_index":"continuous-tests-pods-2023.11.09","_id":"F52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.077280937Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.077+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3138","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224077]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.077545400Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.077+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/TJ/zb2rhb3n6haiuiY5Wngrf3rRzRXT9VwzDsX3fT8GPPwMhiATJ + cid=zb2*MhiATJ count=3139","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224077]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.077987953Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.077+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=1950020 cid=zb2*MhiATJ count=3140","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224077]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Gp2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.084504386Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.084+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3141","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224084]},{"_index":"continuous-tests-pods-2023.11.09","_id":"G52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.084677735Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.084+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*MhiATJ count=3142","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224084]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.084776173Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.084+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=27186 count=3143","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224084]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.084785077Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.084+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3144","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224084]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Hp2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.084896290Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.084+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*MhiATJ queue=0 count=3145","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224084]},{"_index":"continuous-tests-pods-2023.11.09","_id":"I52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.085798946Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.085+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*MhiATJ inflight=1 count=3150","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224085]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.085807257Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.085+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*MhiATJ count=3151","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224085]},{"_index":"continuous-tests-pods-2023.11.09","_id":"H52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.085092669Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.084+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3146","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224085]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.085202954Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.085+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3147","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224085]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.085297749Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.085+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3148","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224085]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ip2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.085544782Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.085+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*MhiATJ count=3149","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224085]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.086118986Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.085+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=28 cid=zb2*MhiATJ bytes=65472 blockOffset=0 count=3152","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224086]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Jp2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.086277772Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.086+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3153","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224086]},{"_index":"continuous-tests-pods-2023.11.09","_id":"J52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.086659457Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.086+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3154","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224086]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.086674196Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.086+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*AjPBrx count=3155","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224086]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Mp2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.087641166Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.087+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3165","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224087]},{"_index":"continuous-tests-pods-2023.11.09","_id":"M52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.087813917Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.087+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3166","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224087]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.087265612Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.087+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*AjPBrx count=3159","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224087]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.087628472Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.087+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*AjPBrx count=3164","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224087]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.087823031Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.087+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3167","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224087]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.087024062Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.086+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*AjPBrx count=3156","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224087]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Kp2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.087232233Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.086+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*AjPBrx peers=1 count=3157","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224087]},{"_index":"continuous-tests-pods-2023.11.09","_id":"K52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.087251538Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.087+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*AjPBrx inFlight=false count=3158","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224087]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.087454659Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.087+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3160","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224087]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Lp2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.087472662Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.087+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3161","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224087]},{"_index":"continuous-tests-pods-2023.11.09","_id":"L52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.087483684Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.087+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*AjPBrx count=3162","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224087]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.087511876Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.087+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*AjPBrx inFlight=true count=3163","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224087]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.088105702Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.087+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*AjPBrx count=3168","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224088]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Np2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.088505983Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.088+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3169","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224088]},{"_index":"continuous-tests-pods-2023.11.09","_id":"N52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.088756155Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.088+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*MhiATJ inflight=0 count=3170","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224088]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.101114386Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.100+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3172","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224101]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Op2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.101803863Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.101+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3173","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224101]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.101079915Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.100+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3171","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224101]},{"_index":"continuous-tests-pods-2023.11.09","_id":"O52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.102117851Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.101+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/rx/zb2rhonVWiStEvSUstUGk4QguXE6ggVU47fbHKqqiN6AjPBrx + cid=zb2*AjPBrx count=3174","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224102]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.102585348Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.102+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=2015492 cid=zb2*AjPBrx count=3175","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224102]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.107855173Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.107+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3176","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224107]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Pp2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.107887293Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.107+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*AjPBrx count=3177","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224107]},{"_index":"continuous-tests-pods-2023.11.09","_id":"P52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.107892651Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.107+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=20591 count=3178","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224107]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.108399051Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.108+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3183","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224108]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.108609167Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.108+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*AjPBrx count=3184","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224108]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Rp2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.108868028Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.108+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*AjPBrx inflight=1 count=3185","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224108]},{"_index":"continuous-tests-pods-2023.11.09","_id":"R52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.108876165Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.108+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*AjPBrx count=3186","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224108]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.108043138Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.107+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3179","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224108]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.108053314Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.107+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*AjPBrx queue=0 count=3180","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224108]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Qp2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.108274925Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.108+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3181","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224108]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Q52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.108285901Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.108+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3182","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224108]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.109154597Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.109+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=29 cid=zb2*AjPBrx bytes=65472 blockOffset=0 count=3187","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224109]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.109260350Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.109+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3188","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224109]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Sp2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.109621657Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.109+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3189","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224109]},{"_index":"continuous-tests-pods-2023.11.09","_id":"S52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.109632852Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.109+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*fQoTEY count=3190","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224109]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.109826879Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.109+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*fQoTEY count=3191","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224109]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.110093603Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.109+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*fQoTEY peers=1 count=3192","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224110]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.110543764Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.110+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3195","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224110]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.110980746Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.110+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3200","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224110]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Tp2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.110226834Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.110+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*fQoTEY inFlight=false count=3193","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224110]},{"_index":"continuous-tests-pods-2023.11.09","_id":"T52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.110421322Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.110+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*fQoTEY count=3194","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224110]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.110619767Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.110+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3196","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224110]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Up2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.110701894Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.110+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*fQoTEY count=3197","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224110]},{"_index":"continuous-tests-pods-2023.11.09","_id":"U52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.110716226Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.110+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*fQoTEY inFlight=true count=3198","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224110]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.110954162Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.110+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*fQoTEY count=3199","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224110]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Vp2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.111215252Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.110+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3201","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224111]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.111406785Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.111+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*fQoTEY count=3203","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224111]},{"_index":"continuous-tests-pods-2023.11.09","_id":"V52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.111229354Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.111+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3202","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224111]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.118560724Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.111+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3204","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224118]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Wp2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.118703212Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.111+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*AjPBrx inflight=0 count=3205","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224118]},{"_index":"continuous-tests-pods-2023.11.09","_id":"W52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.124444906Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.124+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3206","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224124]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.124479212Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.124+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3207","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224124]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.125028511Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.124+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3208","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224125]},{"_index":"continuous-tests-pods-2023.11.09","_id":"X52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.125538008Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.125+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=2080964 cid=zb2*fQoTEY count=3210","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224125]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Xp2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.125108456Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.124+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/EY/zb2rhfgVQn1969L7tEMwrQy2nNT17Xk1SsfcpRp4DtNfQoTEY + cid=zb2*fQoTEY count=3209","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224125]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.130912342Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.130+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3211","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224130]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.131026182Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.130+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*fQoTEY count=3212","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224131]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Yp2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.131060930Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.130+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=20755 count=3213","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224131]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.131487468Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.131+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*fQoTEY count=3219","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224131]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Y52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.131091013Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.130+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3214","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224131]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.131095638Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.130+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*fQoTEY queue=0 count=3215","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224131]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.131102921Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.131+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3216","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224131]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Zp2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.131173241Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.131+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3217","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224131]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Z52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.131194076Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.131+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3218","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224131]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.131666743Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.131+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*fQoTEY inflight=1 count=3220","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224131]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ap2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.131684610Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.131+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*fQoTEY count=3221","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224131]},{"_index":"continuous-tests-pods-2023.11.09","_id":"a52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.132086300Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.131+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=30 cid=zb2*fQoTEY bytes=65472 blockOffset=0 count=3222","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224132]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.132108386Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.131+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3223","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224132]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.132293209Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.132+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3224","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224132]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bp2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.132306432Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.132+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*HrU1JQ count=3225","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224132]},{"_index":"continuous-tests-pods-2023.11.09","_id":"b52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.132529788Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.132+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*HrU1JQ count=3226","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224132]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.132548383Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.132+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*HrU1JQ peers=1 count=3227","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224132]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.132560292Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.132+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*HrU1JQ inFlight=false count=3228","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224132]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cp2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.132658402Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.132+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*HrU1JQ count=3229","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224132]},{"_index":"continuous-tests-pods-2023.11.09","_id":"c52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.132664731Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.132+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3230","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224132]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.132669690Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.132+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3231","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224132]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.132691016Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.132+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*HrU1JQ count=3232","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224132]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dp2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.132829886Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.132+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*HrU1JQ inFlight=true count=3233","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224132]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.132944205Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.132+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3236","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224132]},{"_index":"continuous-tests-pods-2023.11.09","_id":"d52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.132841815Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.132+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*HrU1JQ count=3234","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224132]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.132863861Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.132+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3235","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224132]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ep2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.132954193Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.132+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3237","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224132]},{"_index":"continuous-tests-pods-2023.11.09","_id":"e52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.133232404Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.133+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*HrU1JQ count=3238","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224133]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.134666143Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.134+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3239","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224134]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.134749068Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.134+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*fQoTEY inflight=0 count=3240","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224134]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fp2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.140863920Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.140+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3241","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224140]},{"_index":"continuous-tests-pods-2023.11.09","_id":"f52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.140900738Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.140+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3242","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224140]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.141540013Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.141+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3243","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224141]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.141579756Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.141+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/JQ/zb2rhd5vw5XZn5ti86chFasRi4frD3k6CpLPQkTdSsqHrU1JQ + cid=zb2*HrU1JQ count=3244","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224141]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gp2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.141959865Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.141+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=2146436 cid=zb2*HrU1JQ count=3245","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224141]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hZ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.147536762Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.147+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=14699 count=3248","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224147]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hp2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.147540942Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.147+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3249","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224147]},{"_index":"continuous-tests-pods-2023.11.09","_id":"h52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.147545402Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.147+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*HrU1JQ queue=0 count=3250","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224147]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Gp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.147565835Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.147+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3251","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224147]},{"_index":"continuous-tests-pods-2023.11.09","_id":"G52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.147590807Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.147+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3252","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224147]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.147615025Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.147+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3253","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224147]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.147815068Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.147+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*HrU1JQ count=3254","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224147]},{"_index":"continuous-tests-pods-2023.11.09","_id":"g52EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.147481435Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.147+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3246","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224147]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hJ2EtosBOvK4VyD1la2U","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.147530996Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.147+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*HrU1JQ count=3247","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224147]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.148392633Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.148+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=31 cid=zb2*HrU1JQ bytes=65472 blockOffset=0 count=3257","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224148]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.148464514Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.148+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3258","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224148]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ip2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.148665668Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.148+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3259","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224148]},{"_index":"continuous-tests-pods-2023.11.09","_id":"I52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.148677311Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.148+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*hH8fPf count=3260","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224148]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.148926035Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.148+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*hH8fPf peers=1 count=3262","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224148]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Jp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.148970122Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.148+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*hH8fPf inFlight=false count=3263","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224148]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.148911769Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.148+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*hH8fPf count=3261","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224148]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Hp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.148040841Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.147+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*HrU1JQ inflight=1 count=3255","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224148]},{"_index":"continuous-tests-pods-2023.11.09","_id":"H52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.148050317Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.147+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*HrU1JQ count=3256","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224148]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.149715377Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.149+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*hH8fPf count=3273","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224149]},{"_index":"continuous-tests-pods-2023.11.09","_id":"J52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.149069186Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.148+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*hH8fPf count=3264","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224149]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.149076793Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.149+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3265","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224149]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.149095583Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.149+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3266","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224149]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Kp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.149191157Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.149+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*hH8fPf count=3267","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224149]},{"_index":"continuous-tests-pods-2023.11.09","_id":"K52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.149210056Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.149+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*hH8fPf inFlight=true count=3268","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224149]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.149315272Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.149+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*hH8fPf count=3269","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224149]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.149326644Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.149+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3270","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224149]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Lp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.149418307Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.149+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3271","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224149]},{"_index":"continuous-tests-pods-2023.11.09","_id":"L52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.149430671Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.149+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3272","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224149]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.156099568Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.155+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3274","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224156]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Mp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.156181457Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.155+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*HrU1JQ inflight=0 count=3275","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224156]},{"_index":"continuous-tests-pods-2023.11.09","_id":"M52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.165768405Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.165+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3276","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224165]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.165827298Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.165+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3277","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224165]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.166797740Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.166+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3278","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224166]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Np2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.166912950Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.166+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/Pf/zb2rhiGsPKsdJVKTT7zzu8vzPKB79BA8pVDxweSuuAjhH8fPf + cid=zb2*hH8fPf count=3279","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224166]},{"_index":"continuous-tests-pods-2023.11.09","_id":"N52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.167393534Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.167+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=2211908 cid=zb2*hH8fPf count=3280","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224167]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.173199124Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.172+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3281","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224173]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Op2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.173333377Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.173+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=24182 count=3283","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224173]},{"_index":"continuous-tests-pods-2023.11.09","_id":"O52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.173404964Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.173+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3284","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224173]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Pp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.173443189Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.173+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3287","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224173]},{"_index":"continuous-tests-pods-2023.11.09","_id":"P52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.173485598Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.173+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3288","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224173]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.173907627Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.173+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*hH8fPf inflight=1 count=3290","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224173]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Qp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.173939906Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.173+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*hH8fPf count=3291","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224173]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.173249899Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.172+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*hH8fPf count=3282","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224173]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.173427402Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.173+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3286","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224173]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.173691131Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.173+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*hH8fPf count=3289","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224173]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.173412864Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.173+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*hH8fPf queue=0 count=3285","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224173]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.174275050Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.174+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3293","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224174]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.174850456Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.174+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*D1tCYR peers=1 count=3297","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224174]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Q52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.174204805Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.173+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=32 cid=zb2*hH8fPf bytes=65472 blockOffset=0 count=3292","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224174]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.174647178Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.174+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3294","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224174]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Rp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.174663609Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.174+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*D1tCYR count=3295","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224174]},{"_index":"continuous-tests-pods-2023.11.09","_id":"R52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.174841347Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.174+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*D1tCYR count=3296","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224174]},{"_index":"continuous-tests-pods-2023.11.09","_id":"U52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.175693134Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.175+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*D1tCYR count=3308","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224175]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.175013770Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.174+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*D1tCYR inFlight=false count=3298","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224175]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Sp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.175024016Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.174+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*D1tCYR count=3299","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224175]},{"_index":"continuous-tests-pods-2023.11.09","_id":"S52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.175091815Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.174+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3300","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224175]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.175109527Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.175+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3301","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224175]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.175118960Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.175+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*D1tCYR count=3302","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224175]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Tp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.175244185Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.175+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*D1tCYR inFlight=true count=3303","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224175]},{"_index":"continuous-tests-pods-2023.11.09","_id":"T52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.175252373Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.175+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*D1tCYR count=3304","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224175]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.175261571Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.175+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3305","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224175]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.175402541Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.175+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3306","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224175]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Up2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.175408618Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.175+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3307","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224175]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.178184027Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.177+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3309","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224178]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.178227613Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.177+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*hH8fPf inflight=0 count=3310","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224178]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Vp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.183150917Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.182+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3311","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224183]},{"_index":"continuous-tests-pods-2023.11.09","_id":"V52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.183233887Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.182+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3312","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224183]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.184127097Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.183+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3313","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224184]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.184184691Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.184+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/YR/zb2rho5s6QL38BPF6PoNEMhnB2K1r7BKqMPwnU3rzirD1tCYR + cid=zb2*D1tCYR count=3314","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224184]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Wp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.184801328Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.184+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=2277380 cid=zb2*D1tCYR count=3315","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224184]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.191462570Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.191+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*D1tCYR count=3317","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224191]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.191492345Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.191+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3321","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224191]},{"_index":"continuous-tests-pods-2023.11.09","_id":"W52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.191230737Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.191+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3316","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224191]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.191473962Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.191+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=16503 count=3318","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224191]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Xp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.191479225Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.191+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3319","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224191]},{"_index":"continuous-tests-pods-2023.11.09","_id":"X52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.191483809Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.191+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*D1tCYR queue=0 count=3320","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224191]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.191762510Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.191+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3322","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224191]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Yp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.191775219Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.191+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3323","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224191]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.192162484Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.191+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*D1tCYR inflight=1 count=3325","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224192]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.192167465Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.191+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*D1tCYR count=3326","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224192]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Zp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.192414705Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.192+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=33 cid=zb2*D1tCYR bytes=65472 blockOffset=0 count=3327","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224192]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Z52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.192431627Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.192+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3328","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224192]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Y52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.192098126Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.191+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*D1tCYR count=3324","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224192]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.193132021Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.192+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3329","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224193]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.193149803Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.192+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*uRcyGC count=3330","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224193]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ap2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.193154210Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.192+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*uRcyGC count=3331","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224193]},{"_index":"continuous-tests-pods-2023.11.09","_id":"a52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.193158467Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.192+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*uRcyGC peers=1 count=3332","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224193]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.193162824Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.192+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*uRcyGC inFlight=false count=3333","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224193]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.193167420Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.193+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*uRcyGC count=3334","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224193]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.193211101Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.193+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3335","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224193]},{"_index":"continuous-tests-pods-2023.11.09","_id":"b52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.193228673Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.193+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3336","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224193]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.193239594Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.193+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*uRcyGC count=3337","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224193]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.193292135Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.193+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*uRcyGC inFlight=true count=3338","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224193]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.193319392Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.193+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*uRcyGC count=3339","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224193]},{"_index":"continuous-tests-pods-2023.11.09","_id":"c52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.193325028Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.193+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3340","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224193]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.193378101Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.193+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3341","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224193]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.193476850Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.193+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3342","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224193]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.193667369Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.193+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*uRcyGC count=3343","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224193]},{"_index":"continuous-tests-pods-2023.11.09","_id":"d52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.199643797Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.199+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3344","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224199]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.199803702Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.199+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*D1tCYR inflight=0 count=3345","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224199]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.204739517Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.204+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3346","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224204]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ep2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.204815804Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.204+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3347","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224204]},{"_index":"continuous-tests-pods-2023.11.09","_id":"e52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.205425559Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.205+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3348","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224205]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.205614988Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.205+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/GC/zb2rhjwo57VGGWqHmuX5wJZHjXyx1KzJAiScqE6NBTjuRcyGC + cid=zb2*uRcyGC count=3349","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224205]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.206006049Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.205+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=2342852 cid=zb2*uRcyGC count=3350","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224206]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.211900494Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.211+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=18728 count=3353","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224211]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.211905131Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.211+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3354","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224211]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.211909606Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.211+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*uRcyGC queue=0 count=3355","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224211]},{"_index":"continuous-tests-pods-2023.11.09","_id":"g52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.211914038Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.211+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3356","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224211]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.211844426Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.211+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3351","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224211]},{"_index":"continuous-tests-pods-2023.11.09","_id":"f52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.211893851Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.211+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*uRcyGC count=3352","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224211]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.211952414Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.211+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3357","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224211]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.211981280Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.211+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3358","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224211]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.212143731Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.211+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*uRcyGC count=3359","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224212]},{"_index":"continuous-tests-pods-2023.11.09","_id":"h52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.212200940Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.212+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*uRcyGC inflight=1 count=3360","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224212]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.212249573Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.212+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*uRcyGC count=3361","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224212]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.212512625Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.212+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=34 cid=zb2*uRcyGC bytes=65472 blockOffset=0 count=3362","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224212]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ip2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.212533087Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.212+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3363","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224212]},{"_index":"continuous-tests-pods-2023.11.09","_id":"i52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.212839588Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.212+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3364","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224212]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.212849068Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.212+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*7NbHWE count=3365","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224212]},{"_index":"continuous-tests-pods-2023.11.09","_id":"j52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.213167660Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.213+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*7NbHWE inFlight=false count=3368","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224213]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.213198463Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.213+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*7NbHWE count=3369","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224213]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.213066461Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.212+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*7NbHWE count=3366","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224213]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.213076815Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.212+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*7NbHWE peers=1 count=3367","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224213]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.213203292Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.213+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3370","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224213]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.213411132Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.213+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3371","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224213]},{"_index":"continuous-tests-pods-2023.11.09","_id":"k52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.213433748Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.213+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*7NbHWE count=3372","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224213]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.213496268Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.213+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*7NbHWE inFlight=true count=3373","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224213]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.213509297Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.213+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*7NbHWE count=3374","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224213]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.213514259Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.213+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3375","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224213]},{"_index":"continuous-tests-pods-2023.11.09","_id":"l52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.213518767Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.213+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3376","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224213]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.213528310Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.213+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3377","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224213]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.213769685Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.213+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*7NbHWE count=3378","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224213]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.218844102Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.218+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3379","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224218]},{"_index":"continuous-tests-pods-2023.11.09","_id":"m52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.218972002Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.218+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*uRcyGC inflight=0 count=3380","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224218]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.222538327Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.222+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3381","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224222]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.222607960Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.222+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3382","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224222]},{"_index":"continuous-tests-pods-2023.11.09","_id":"np2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.223240238Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.222+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3383","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224223]},{"_index":"continuous-tests-pods-2023.11.09","_id":"n52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.223342041Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.223+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/WE/zb2rhabRZ9hcqDxN3erR2ykaNPTyKiKVuJPBVA7e82i7NbHWE + cid=zb2*7NbHWE count=3384","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224223]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.223799661Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.223+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=2408324 cid=zb2*7NbHWE count=3385","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224223]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.229356642Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.228+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3386","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224229]},{"_index":"continuous-tests-pods-2023.11.09","_id":"op2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.229389351Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.228+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*7NbHWE count=3387","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224229]},{"_index":"continuous-tests-pods-2023.11.09","_id":"o52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.229394458Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.229+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=15994 count=3388","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224229]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.229402425Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.229+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*7NbHWE queue=0 count=3390","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224229]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.229424965Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.229+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3391","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224229]},{"_index":"continuous-tests-pods-2023.11.09","_id":"p52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.229429290Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.229+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3392","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224229]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.229440955Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.229+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3393","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224229]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.229759798Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.229+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*7NbHWE inflight=1 count=3395","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224229]},{"_index":"continuous-tests-pods-2023.11.09","_id":"q52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.229783916Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.229+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*7NbHWE count=3396","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224229]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.229398416Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.229+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3389","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224229]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.229600675Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.229+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*7NbHWE count=3394","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224229]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.230027530Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.229+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=35 cid=zb2*7NbHWE bytes=65472 blockOffset=0 count=3397","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224230]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.230194166Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.230+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3399","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224230]},{"_index":"continuous-tests-pods-2023.11.09","_id":"r52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.230213624Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.230+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*n6awhp count=3400","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224230]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.230421844Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.230+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*n6awhp count=3401","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224230]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.230432729Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.230+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*n6awhp peers=1 count=3402","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224230]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.230461061Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.230+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*n6awhp inFlight=false count=3403","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224230]},{"_index":"continuous-tests-pods-2023.11.09","_id":"s52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.230716185Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.230+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*n6awhp count=3404","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224230]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.230730333Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.230+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3405","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224230]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.230748508Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.230+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3406","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224230]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.230753212Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.230+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*n6awhp count=3407","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224230]},{"_index":"continuous-tests-pods-2023.11.09","_id":"t52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.230758290Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.230+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*n6awhp inFlight=true count=3408","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224230]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.230796494Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.230+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*n6awhp count=3409","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224230]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.230803848Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.230+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3410","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224230]},{"_index":"continuous-tests-pods-2023.11.09","_id":"up2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.230869108Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.230+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3411","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224230]},{"_index":"continuous-tests-pods-2023.11.09","_id":"u52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.230987456Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.230+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3412","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224230]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.230047128Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.229+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3398","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224230]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.231138250Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.231+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*n6awhp count=3413","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224231]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.243236635Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.242+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3414","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224243]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.243299375Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.243+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*7NbHWE inflight=0 count=3415","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224243]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.245282853Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.245+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3417","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224245]},{"_index":"continuous-tests-pods-2023.11.09","_id":"v52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.245229306Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.244+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3416","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224245]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.246070454Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.245+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3418","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224246]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.246161009Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.246+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/hp/zb2rhdfjXfWs7QrkMZdS2NGtZRyriJQ6nbQXJugyuUKn6awhp + cid=zb2*n6awhp count=3419","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224246]},{"_index":"continuous-tests-pods-2023.11.09","_id":"w52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.246655732Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.246+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=2473796 cid=zb2*n6awhp count=3420","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224246]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.252777964Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.252+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=22110 count=3423","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224252]},{"_index":"continuous-tests-pods-2023.11.09","_id":"x52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.252787425Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.252+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3424","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224252]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.252881072Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.252+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3427","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224252]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.252674277Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.252+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3421","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224252]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.252709717Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.252+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*n6awhp count=3422","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224252]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.252837329Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.252+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*n6awhp queue=0 count=3425","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224252]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.252844966Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.252+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3426","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224252]},{"_index":"continuous-tests-pods-2023.11.09","_id":"y52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.253020280Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.252+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3428","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224253]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zJ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.253165462Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.253+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*n6awhp count=3429","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224253]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zZ2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.253370314Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.253+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*n6awhp inflight=1 count=3430","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224253]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zp2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.253408135Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.253+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*n6awhp count=3431","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224253]},{"_index":"continuous-tests-pods-2023.11.09","_id":"z52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.253634428Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.253+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=36 cid=zb2*n6awhp bytes=65472 blockOffset=0 count=3432","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224253]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0J2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.253685973Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.253+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3433","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224253]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0Z2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.253928712Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.253+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3434","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224253]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0p2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.253942636Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.253+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*P35Zd5 count=3435","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224253]},{"_index":"continuous-tests-pods-2023.11.09","_id":"252EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.254584313Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.254+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*P35Zd5 count=3444","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224254]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3J2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.254601053Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.254+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3445","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224254]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3Z2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.254650767Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.254+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3446","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224254]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3p2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.254657735Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.254+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3447","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224254]},{"_index":"continuous-tests-pods-2023.11.09","_id":"352EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.254866238Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.254+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*P35Zd5 count=3448","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224254]},{"_index":"continuous-tests-pods-2023.11.09","_id":"152EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.254477717Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.254+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3440","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224254]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2J2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.254506375Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.254+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3441","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224254]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2Z2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.254512271Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.254+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*P35Zd5 count=3442","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224254]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2p2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.254523011Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.254+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*P35Zd5 inFlight=true count=3443","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224254]},{"_index":"continuous-tests-pods-2023.11.09","_id":"052EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.254136590Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.254+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*P35Zd5 count=3436","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224254]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1J2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.254154506Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.254+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*P35Zd5 peers=1 count=3437","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224254]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1Z2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.254205560Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.254+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*P35Zd5 inFlight=false count=3438","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224254]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1p2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.254216237Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.254+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*P35Zd5 count=3439","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224254]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4J2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.261705794Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.261+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3449","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224261]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4Z2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.261740955Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.261+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*n6awhp inflight=0 count=3450","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224261]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4p2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.265198598Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.264+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3451","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224265]},{"_index":"continuous-tests-pods-2023.11.09","_id":"452EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.265244115Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.265+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3452","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224265]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5J2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.266016428Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.265+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3453","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224266]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5Z2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.266024297Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.265+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/d5/zb2rhgaQXfRYneMvGArb92m6eWdWntQe1KJkcmv2ZNjP35Zd5 + cid=zb2*P35Zd5 count=3454","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224266]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5p2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.266644299Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.266+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=2539268 cid=zb2*P35Zd5 count=3455","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224266]},{"_index":"continuous-tests-pods-2023.11.09","_id":"552EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.271826303Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.271+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3456","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224271]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6J2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.271859766Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.271+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*P35Zd5 count=3457","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224271]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6Z2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.271891015Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.271+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=17693 count=3458","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224271]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6p2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.271895256Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.271+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3459","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224271]},{"_index":"continuous-tests-pods-2023.11.09","_id":"652EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.271972833Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.271+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*P35Zd5 queue=0 count=3460","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224271]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7J2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.271983182Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.271+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3461","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224271]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7Z2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.272159065Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.271+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3462","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224272]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7p2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.272182684Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.272+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3463","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224272]},{"_index":"continuous-tests-pods-2023.11.09","_id":"752EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.272283159Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.272+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*P35Zd5 count=3464","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224272]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8J2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.272433447Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.272+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*P35Zd5 inflight=1 count=3465","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224272]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8Z2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.272540954Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.272+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*P35Zd5 count=3466","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224272]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8p2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.272770499Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.272+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=37 cid=zb2*P35Zd5 bytes=65472 blockOffset=0 count=3467","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224272]},{"_index":"continuous-tests-pods-2023.11.09","_id":"852EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.272801980Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.272+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3468","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224272]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_Z2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.273714424Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.273+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*LTfXyx inFlight=true count=3478","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224273]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9J2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.273085373Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.272+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3469","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224273]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9Z2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.273188933Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.273+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*LTfXyx count=3470","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224273]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9p2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.273321682Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.273+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*LTfXyx count=3471","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224273]},{"_index":"continuous-tests-pods-2023.11.09","_id":"952EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.273347872Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.273+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*LTfXyx peers=1 count=3472","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224273]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-J2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.273464491Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.273+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*LTfXyx inFlight=false count=3473","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224273]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-Z2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.273480372Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.273+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*LTfXyx count=3474","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224273]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-p2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.273521443Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.273+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3475","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224273]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.273543664Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.273+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3476","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224273]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_J2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.273598292Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.273+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*LTfXyx count=3477","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224273]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_p2EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.273750496Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.273+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*LTfXyx count=3479","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224273]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_52EtosBOvK4VyD1lrJ7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.273766392Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.273+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3480","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224273]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.273836635Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.273+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3481","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224273]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.274015356Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.273+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3482","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224274]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ap2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.274089162Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.274+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*LTfXyx count=3483","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224274]},{"_index":"continuous-tests-pods-2023.11.09","_id":"A52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.282103189Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.281+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3484","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224282]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.282142138Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.281+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*P35Zd5 inflight=0 count=3485","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224282]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.287469880Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.287+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3486","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224287]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Bp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.287519211Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.287+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3487","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224287]},{"_index":"continuous-tests-pods-2023.11.09","_id":"B52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.288069775Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.287+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3488","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224288]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.288344002Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.288+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/yx/zb2rhkBzNGsyzjXX6MB37bC9bRfbUsS9xe8yrHzHKoLLTfXyx + cid=zb2*LTfXyx count=3489","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224288]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.288584449Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.288+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=2604740 cid=zb2*LTfXyx count=3490","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224288]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Cp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.304285736Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.303+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3491","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224304]},{"_index":"continuous-tests-pods-2023.11.09","_id":"C52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.304327792Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.304+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*LTfXyx count=3492","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224304]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.304333786Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.304+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=30809 count=3493","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224304]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.304338431Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.304+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3494","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224304]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Dp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.304466921Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.304+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*LTfXyx queue=0 count=3495","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224304]},{"_index":"continuous-tests-pods-2023.11.09","_id":"D52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.304527154Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.304+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3496","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224304]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.304534649Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.304+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3497","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224304]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.304539943Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.304+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3498","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224304]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ep2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.304772277Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.304+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*LTfXyx count=3499","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224304]},{"_index":"continuous-tests-pods-2023.11.09","_id":"E52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.304966335Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.304+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*LTfXyx inflight=1 count=3500","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224304]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.305027873Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.304+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*LTfXyx count=3501","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224305]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.305179418Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.305+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=38 cid=zb2*LTfXyx bytes=65472 blockOffset=0 count=3502","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224305]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Fp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.305326624Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.305+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3503","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224305]},{"_index":"continuous-tests-pods-2023.11.09","_id":"F52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.305601848Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.305+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3504","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224305]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.305612343Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.305+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*frXdG3 count=3505","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224305]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.305825062Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.305+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*frXdG3 count=3506","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224305]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Gp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.305834599Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.305+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*frXdG3 peers=1 count=3507","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224305]},{"_index":"continuous-tests-pods-2023.11.09","_id":"G52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.305886831Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.305+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*frXdG3 inFlight=false count=3508","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224305]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.305920068Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.305+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*frXdG3 count=3509","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224305]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.306038644Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.305+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3510","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224306]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Hp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.306048563Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.305+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3511","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224306]},{"_index":"continuous-tests-pods-2023.11.09","_id":"H52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.306059091Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.305+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*frXdG3 count=3512","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224306]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.306105495Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.306+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*frXdG3 inFlight=true count=3513","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224306]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.306148805Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.306+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*frXdG3 count=3514","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224306]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ip2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.306251188Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.306+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3515","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224306]},{"_index":"continuous-tests-pods-2023.11.09","_id":"I52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.306306957Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.306+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3516","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224306]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.306449033Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.306+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3517","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224306]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.306613244Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.306+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*frXdG3 count=3518","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224306]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Jp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.316762189Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.316+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3519","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224316]},{"_index":"continuous-tests-pods-2023.11.09","_id":"J52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.316795815Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.316+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*LTfXyx inflight=0 count=3520","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224316]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.318718643Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.318+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3521","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224318]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.318756689Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.318+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3522","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224318]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Kp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.319832435Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.319+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3523","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224319]},{"_index":"continuous-tests-pods-2023.11.09","_id":"K52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.319872305Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.319+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/G3/zb2rhYDBuvbDNJqrLfH8dsr8xEdoZVsBiW712Y6BbXFfrXdG3 + cid=zb2*frXdG3 count=3524","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224319]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.320380980Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.320+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=2670212 cid=zb2*frXdG3 count=3525","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224320]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.325944007Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.325+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3526","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224325]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Lp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.325983862Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.325+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*frXdG3 count=3527","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224325]},{"_index":"continuous-tests-pods-2023.11.09","_id":"L52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.326056371Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.325+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=20009 count=3528","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224326]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.326083564Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.325+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3529","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224326]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.326123263Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.325+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*frXdG3 queue=0 count=3530","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224326]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Mp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.326145054Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.326+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3531","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224326]},{"_index":"continuous-tests-pods-2023.11.09","_id":"M52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.326233187Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.326+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3532","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224326]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.326246733Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.326+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3533","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224326]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.326527167Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.326+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*frXdG3 count=3534","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224326]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Np2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.326653454Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.326+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*frXdG3 inflight=1 count=3535","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224326]},{"_index":"continuous-tests-pods-2023.11.09","_id":"N52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.326734298Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.326+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*frXdG3 count=3536","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224326]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.326922821Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.326+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=39 cid=zb2*frXdG3 bytes=65472 blockOffset=0 count=3537","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224326]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.326990135Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.326+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3538","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224326]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Op2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.327262990Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.327+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3539","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224327]},{"_index":"continuous-tests-pods-2023.11.09","_id":"O52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.327289010Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.327+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*NJpwYq count=3540","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224327]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.327425369Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.327+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*NJpwYq count=3541","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224327]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.327437508Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.327+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*NJpwYq peers=1 count=3542","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224327]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Pp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.327553675Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.327+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*NJpwYq inFlight=false count=3543","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224327]},{"_index":"continuous-tests-pods-2023.11.09","_id":"P52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.327591983Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.327+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*NJpwYq count=3544","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224327]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.327650108Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.327+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3545","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224327]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.327658106Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.327+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3546","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224327]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Qp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.327688268Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.327+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*NJpwYq count=3547","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224327]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Q52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.327775836Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.327+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*NJpwYq inFlight=true count=3548","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224327]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.327817022Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.327+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*NJpwYq count=3549","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224327]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.327823465Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.327+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3550","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224327]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Rp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.327866528Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.327+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3551","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224327]},{"_index":"continuous-tests-pods-2023.11.09","_id":"R52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.327979686Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.327+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3552","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224327]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.328120344Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.328+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*NJpwYq count=3553","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224328]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.328469056Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.328+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3554","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224328]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Sp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.328604857Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.328+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*frXdG3 inflight=0 count=3555","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224328]},{"_index":"continuous-tests-pods-2023.11.09","_id":"S52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.348761860Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.348+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3556","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224348]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.348802216Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.348+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3557","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224348]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.349898649Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.349+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3558","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224349]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Tp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.349961681Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.349+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/Yq/zb2rhfva8wgMpoGHaqmhCMcDkcpbEzkaT4EovT1worFNJpwYq + cid=zb2*NJpwYq count=3559","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224349]},{"_index":"continuous-tests-pods-2023.11.09","_id":"T52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.350395122Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.350+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=2735684 cid=zb2*NJpwYq count=3560","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224350]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.356844391Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.356+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3561","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224356]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.356885874Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.356+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*NJpwYq count=3562","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224356]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Up2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.356932239Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.356+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=29298 count=3563","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224356]},{"_index":"continuous-tests-pods-2023.11.09","_id":"U52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.356937772Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.356+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3564","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224356]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.356941994Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.356+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*NJpwYq queue=0 count=3565","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224356]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.356964747Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.356+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3566","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224356]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Vp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.356987815Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.356+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3567","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224356]},{"_index":"continuous-tests-pods-2023.11.09","_id":"V52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.357012149Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.356+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3568","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224357]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.357181797Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.357+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*NJpwYq count=3569","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224357]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.357419211Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.357+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*NJpwYq inflight=1 count=3570","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224357]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Wp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.357482530Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.357+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*NJpwYq count=3571","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224357]},{"_index":"continuous-tests-pods-2023.11.09","_id":"W52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.357615396Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.357+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=40 cid=zb2*NJpwYq bytes=65472 blockOffset=0 count=3572","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224357]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.357659123Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.357+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3573","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224357]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.358609461Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.358+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*LtCwtp count=3582","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224358]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Zp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.358627355Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.358+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*LtCwtp inFlight=true count=3583","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224358]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Z52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.358631995Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.358+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*LtCwtp count=3584","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224358]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.358642570Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.358+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3585","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224358]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.358752631Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.358+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3586","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224358]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.358003246Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.357+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3574","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224358]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Xp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.358019519Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.357+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*LtCwtp count=3575","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224358]},{"_index":"continuous-tests-pods-2023.11.09","_id":"X52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.358092737Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.358+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*LtCwtp count=3576","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224358]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.358213589Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.358+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*LtCwtp peers=1 count=3577","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224358]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.358238267Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.358+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*LtCwtp inFlight=false count=3578","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224358]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Yp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.358274519Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.358+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*LtCwtp count=3579","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224358]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Y52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.358415274Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.358+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3580","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224358]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.358426381Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.358+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3581","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224358]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ap2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.358772106Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.358+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3587","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224358]},{"_index":"continuous-tests-pods-2023.11.09","_id":"a52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.359071411Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.358+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*LtCwtp count=3588","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224359]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.363851631Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.363+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3589","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224363]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.363918513Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.363+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*NJpwYq inflight=0 count=3590","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224363]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.368467425Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.368+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3591","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224368]},{"_index":"continuous-tests-pods-2023.11.09","_id":"b52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.368533886Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.368+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3592","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224368]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.369422620Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.369+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3593","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224369]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.369448919Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.369+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/tp/zb2rhfB9A5LQi9aZAcRBhquJH67L3fsPQSXBgbG479WLtCwtp + cid=zb2*LtCwtp count=3594","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224369]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.370171519Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.369+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=2801156 cid=zb2*LtCwtp count=3595","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224370]},{"_index":"continuous-tests-pods-2023.11.09","_id":"d52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.376285952Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.376+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*LtCwtp queue=0 count=3600","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224376]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.376293454Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.376+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3601","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224376]},{"_index":"continuous-tests-pods-2023.11.09","_id":"c52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.376124841Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.375+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3596","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224376]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.376161846Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.375+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*LtCwtp count=3597","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224376]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.376167902Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.375+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=17842 count=3598","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224376]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.376231969Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.376+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3599","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224376]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.376337102Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.376+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3602","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224376]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ep2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.376369336Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.376+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3603","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224376]},{"_index":"continuous-tests-pods-2023.11.09","_id":"e52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.376652506Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.376+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*LtCwtp count=3604","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224376]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.376853176Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.376+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*LtCwtp inflight=1 count=3605","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224376]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.376888821Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.376+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*LtCwtp count=3606","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224376]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.377220285Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.376+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=41 cid=zb2*LtCwtp bytes=65472 blockOffset=0 count=3607","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224377]},{"_index":"continuous-tests-pods-2023.11.09","_id":"f52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.377261476Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.377+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3608","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224377]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.377524654Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.377+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3609","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224377]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.377574996Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.377+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*SvAy8G count=3610","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224377]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.377782144Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.377+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*SvAy8G count=3611","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224377]},{"_index":"continuous-tests-pods-2023.11.09","_id":"g52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.377839632Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.377+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*SvAy8G peers=1 count=3612","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224377]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.377846673Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.377+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*SvAy8G inFlight=false count=3613","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224377]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.378249950Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.378+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3621","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224378]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.378272668Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.378+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3622","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224378]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.378655395Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.378+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*SvAy8G count=3623","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224378]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.378008800Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.377+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*SvAy8G count=3614","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224378]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.378044360Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.377+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3615","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224378]},{"_index":"continuous-tests-pods-2023.11.09","_id":"h52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.378051003Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.377+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3616","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224378]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.378055454Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.377+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*SvAy8G count=3617","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224378]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.378060005Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.377+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*SvAy8G inFlight=true count=3618","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224378]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ip2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.378194148Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.378+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*SvAy8G count=3619","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224378]},{"_index":"continuous-tests-pods-2023.11.09","_id":"i52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.378224454Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.378+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3620","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224378]},{"_index":"continuous-tests-pods-2023.11.09","_id":"j52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.379353825Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.379+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3624","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224379]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.379412008Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.379+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*LtCwtp inflight=0 count=3625","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224379]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.384454354Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.384+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3626","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224384]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.384498198Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.384+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3627","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224384]},{"_index":"continuous-tests-pods-2023.11.09","_id":"k52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.385475403Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.385+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3628","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224385]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.385571663Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.385+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/8G/zb2rhmMCbc1VepkKdR5DxschhzUVH7khUKto93hAh4HSvAy8G + cid=zb2*SvAy8G count=3629","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224385]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.386104893Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.385+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=2866628 cid=zb2*SvAy8G count=3630","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224386]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.391773845Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.391+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*SvAy8G queue=0 count=3635","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224391]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.391547129Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.391+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3631","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224391]},{"_index":"continuous-tests-pods-2023.11.09","_id":"l52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.391578384Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.391+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*SvAy8G count=3632","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224391]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.391735572Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.391+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=13686 count=3633","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224391]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.391759408Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.391+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3634","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224391]},{"_index":"continuous-tests-pods-2023.11.09","_id":"m52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.391822761Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.391+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3636","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224391]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.391839476Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.391+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3637","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224391]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.391917334Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.391+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3638","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224391]},{"_index":"continuous-tests-pods-2023.11.09","_id":"op2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.392743215Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.392+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3643","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224392]},{"_index":"continuous-tests-pods-2023.11.09","_id":"o52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.392999559Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.392+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3644","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224392]},{"_index":"continuous-tests-pods-2023.11.09","_id":"np2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.392096640Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.391+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*SvAy8G count=3639","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224392]},{"_index":"continuous-tests-pods-2023.11.09","_id":"n52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.392265191Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.392+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*SvAy8G inflight=1 count=3640","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224392]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.392317737Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.392+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*SvAy8G count=3641","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224392]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.392728494Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.392+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=42 cid=zb2*SvAy8G bytes=65472 blockOffset=0 count=3642","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224392]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.393292519Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.393+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3650","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224393]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.393303267Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.393+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3651","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224393]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Op2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.393308772Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.393+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*UivE9c count=3652","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224393]},{"_index":"continuous-tests-pods-2023.11.09","_id":"O52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.393501866Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.393+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*UivE9c inFlight=true count=3653","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224393]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.393520120Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.393+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*UivE9c count=3654","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224393]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.393599898Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.393+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3655","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224393]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Pp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.393617837Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.393+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3656","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224393]},{"_index":"continuous-tests-pods-2023.11.09","_id":"P52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.393623146Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.393+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3657","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224393]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.393839273Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.393+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*UivE9c count=3658","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224393]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pZ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.393064326Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.392+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*UivE9c count=3646","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224393]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pp2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.393072278Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.393+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*UivE9c peers=1 count=3647","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224393]},{"_index":"continuous-tests-pods-2023.11.09","_id":"p52EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.393119504Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.393+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*UivE9c inFlight=false count=3648","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224393]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.393257814Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.393+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*UivE9c count=3649","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224393]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pJ2EtosBOvK4VyD1lrN7","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.393012085Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.392+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*UivE9c count=3645","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224393]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.394440566Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.394+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3659","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224394]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Qp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.394465458Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.394+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*SvAy8G inflight=0 count=3660","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224394]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Q52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.408048755Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.407+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3661","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224408]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.408081884Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.407+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3662","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224408]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.408625686Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.408+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3663","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224408]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Rp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.408750174Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.408+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/9c/zb2rhcq5HxgoVdAwNk8aMijUtYzDowzh48aKJ6HDvFaUivE9c + cid=zb2*UivE9c count=3664","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224408]},{"_index":"continuous-tests-pods-2023.11.09","_id":"R52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.409274510Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.409+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=2932100 cid=zb2*UivE9c count=3665","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224409]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.415316963Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.414+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3666","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224415]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.415353034Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.415+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*UivE9c count=3667","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224415]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Sp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.415375005Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.415+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=22083 count=3668","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224415]},{"_index":"continuous-tests-pods-2023.11.09","_id":"S52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.415399461Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.415+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3669","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224415]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.415406472Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.415+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*UivE9c queue=0 count=3670","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224415]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.415560529Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.415+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3671","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224415]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Tp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.415579354Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.415+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3672","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224415]},{"_index":"continuous-tests-pods-2023.11.09","_id":"T52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.415592165Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.415+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3673","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224415]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.415759519Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.415+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*UivE9c count=3674","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224415]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.415946582Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.415+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*UivE9c inflight=1 count=3675","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224415]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Up2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.416020929Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.415+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*UivE9c count=3676","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224416]},{"_index":"continuous-tests-pods-2023.11.09","_id":"U52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.416230253Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.416+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=43 cid=zb2*UivE9c bytes=65472 blockOffset=0 count=3677","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224416]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.416241600Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.416+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3678","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224416]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.416520171Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.416+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3679","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224416]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Vp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.416529638Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.416+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*pCd7Pq count=3680","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224416]},{"_index":"continuous-tests-pods-2023.11.09","_id":"V52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.416640186Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.416+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*pCd7Pq count=3681","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224416]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.416649877Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.416+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*pCd7Pq peers=1 count=3682","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224416]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.416776397Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.416+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*pCd7Pq inFlight=false count=3683","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224416]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Wp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.416929389Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.416+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*pCd7Pq count=3684","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224416]},{"_index":"continuous-tests-pods-2023.11.09","_id":"W52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.416940501Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.416+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3685","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224416]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.416945505Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.416+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3686","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224416]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.416949756Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.416+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*pCd7Pq count=3687","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224416]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Xp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.416954211Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.416+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*pCd7Pq inFlight=true count=3688","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224416]},{"_index":"continuous-tests-pods-2023.11.09","_id":"X52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.416969400Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.416+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*pCd7Pq count=3689","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224416]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.417036939Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.416+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3690","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224417]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.417044892Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.416+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3691","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224417]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Yp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.417144410Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.417+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3692","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224417]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Y52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.417316824Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.417+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*pCd7Pq count=3693","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224417]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.417591003Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.417+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3694","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224417]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.417769294Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.417+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*UivE9c inflight=0 count=3695","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224417]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Zp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.431095278Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.430+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3696","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224431]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Z52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.431139402Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.430+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3697","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224431]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.431726420Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.431+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3698","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224431]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.431874967Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.431+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/Pq/zb2rhgiVPTvx8DagXp6QzyTvHQ16vxFmK55kYTjRuXTpCd7Pq + cid=zb2*pCd7Pq count=3699","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224431]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ap2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.432386702Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.432+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=2997572 cid=zb2*pCd7Pq count=3700","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224432]},{"_index":"continuous-tests-pods-2023.11.09","_id":"a52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.438086679Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.437+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3701","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224438]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.438159655Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.437+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*pCd7Pq count=3702","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224438]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.438203330Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.438+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=21409 count=3703","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224438]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.438209183Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.438+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3704","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224438]},{"_index":"continuous-tests-pods-2023.11.09","_id":"b52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.438213856Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.438+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*pCd7Pq queue=0 count=3705","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224438]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.438264571Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.438+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3706","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224438]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.438380341Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.438+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3707","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224438]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.438436145Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.438+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3708","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224438]},{"_index":"continuous-tests-pods-2023.11.09","_id":"c52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.438662569Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.438+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*pCd7Pq count=3709","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224438]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.438817234Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.438+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*pCd7Pq inflight=1 count=3710","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224438]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.438948276Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.438+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*pCd7Pq count=3711","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224438]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.439214256Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.438+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=44 cid=zb2*pCd7Pq bytes=65472 blockOffset=0 count=3712","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224439]},{"_index":"continuous-tests-pods-2023.11.09","_id":"d52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.439233426Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.439+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3713","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224439]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.439326645Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.439+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3714","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224439]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.439454254Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.439+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*ffAhpZ count=3715","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224439]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ep2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.439588742Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.439+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*ffAhpZ count=3716","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224439]},{"_index":"continuous-tests-pods-2023.11.09","_id":"e52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.439681956Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.439+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*ffAhpZ peers=1 count=3717","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224439]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.439715051Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.439+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*ffAhpZ inFlight=false count=3718","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224439]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.439780517Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.439+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*ffAhpZ count=3719","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224439]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.439809302Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.439+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3720","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224439]},{"_index":"continuous-tests-pods-2023.11.09","_id":"f52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.439859337Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.439+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3721","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224439]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.439946620Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.439+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*ffAhpZ count=3722","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224439]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.440005347Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.439+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*ffAhpZ inFlight=true count=3723","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224440]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.440012572Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.439+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*ffAhpZ count=3724","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224440]},{"_index":"continuous-tests-pods-2023.11.09","_id":"g52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.440133126Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.439+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3725","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224440]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.440189264Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.440+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3726","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224440]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.440200082Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.440+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3727","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224440]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.440346793Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.440+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*ffAhpZ count=3728","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224440]},{"_index":"continuous-tests-pods-2023.11.09","_id":"h52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.441715893Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.441+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3729","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224441]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.441767605Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.441+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*pCd7Pq inflight=0 count=3730","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224441]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.458184109Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.457+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3731","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224458]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ip2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.458218999Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.457+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3732","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224458]},{"_index":"continuous-tests-pods-2023.11.09","_id":"i52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.459065135Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.458+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3733","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224459]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.459314210Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.459+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/pZ/zb2rhnKCr1k6XSKhpyDsnioizDe8fuCdo8M14LDCoWTffAhpZ + cid=zb2*ffAhpZ count=3734","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224459]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.459860938Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.459+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=3063044 cid=zb2*ffAhpZ count=3735","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224459]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.465921944Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.465+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3736","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224465]},{"_index":"continuous-tests-pods-2023.11.09","_id":"j52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.465963799Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.465+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*ffAhpZ count=3737","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224465]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.465977630Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.465+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=26139 count=3738","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224465]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.466053666Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.465+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3739","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224466]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.466086372Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.465+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*ffAhpZ queue=0 count=3740","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224466]},{"_index":"continuous-tests-pods-2023.11.09","_id":"k52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.466125244Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.466+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3741","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224466]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.466132581Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.466+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3742","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224466]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.466151607Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.466+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3743","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224466]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.466476577Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.466+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*ffAhpZ count=3744","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224466]},{"_index":"continuous-tests-pods-2023.11.09","_id":"l52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.466651898Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.466+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*ffAhpZ inflight=1 count=3745","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224466]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.466732209Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.466+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*ffAhpZ count=3746","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224466]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.466979474Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.466+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=45 cid=zb2*ffAhpZ bytes=65472 blockOffset=0 count=3747","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224466]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.467023366Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.466+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3748","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224467]},{"_index":"continuous-tests-pods-2023.11.09","_id":"m52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.467263663Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.467+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3749","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224467]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.467274916Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.467+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*vbt17w count=3750","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224467]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.467462235Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.467+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*vbt17w count=3751","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224467]},{"_index":"continuous-tests-pods-2023.11.09","_id":"np2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.467476264Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.467+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*vbt17w peers=1 count=3752","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224467]},{"_index":"continuous-tests-pods-2023.11.09","_id":"n52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.467524512Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.467+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*vbt17w inFlight=false count=3753","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224467]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.467624509Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.467+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*vbt17w count=3754","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224467]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.467637207Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.467+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3755","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224467]},{"_index":"continuous-tests-pods-2023.11.09","_id":"op2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.467679769Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.467+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3756","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224467]},{"_index":"continuous-tests-pods-2023.11.09","_id":"o52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.467701107Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.467+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*vbt17w count=3757","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224467]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.467747768Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.467+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*vbt17w inFlight=true count=3758","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224467]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.467844668Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.467+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*vbt17w count=3759","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224467]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.467870835Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.467+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3760","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224467]},{"_index":"continuous-tests-pods-2023.11.09","_id":"p52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.467919185Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.467+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3761","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224467]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.467984776Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.467+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3762","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224467]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.468139549Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.468+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*vbt17w count=3763","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224468]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.468692211Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.468+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3764","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224468]},{"_index":"continuous-tests-pods-2023.11.09","_id":"q52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.468839253Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.468+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*ffAhpZ inflight=0 count=3765","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224468]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.480668447Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.480+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3766","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224480]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.480704500Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.480+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3767","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224480]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.481241837Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.481+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3768","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224481]},{"_index":"continuous-tests-pods-2023.11.09","_id":"r52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.481359723Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.481+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/7w/zb2rhaHYyLU8wFEf6WjoEQkZhPYDgzuqPJBEqmpYGfuvbt17w + cid=zb2*vbt17w count=3769","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224481]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.482284817Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.481+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=3128516 cid=zb2*vbt17w count=3770","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224482]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.487171337Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.486+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3771","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224487]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.487202779Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.487+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*vbt17w count=3772","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224487]},{"_index":"continuous-tests-pods-2023.11.09","_id":"s52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.487207983Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.487+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=19598 count=3773","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224487]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.487297685Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.487+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3774","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224487]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.487303697Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.487+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*vbt17w queue=0 count=3775","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224487]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.487359907Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.487+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3776","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224487]},{"_index":"continuous-tests-pods-2023.11.09","_id":"t52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.487368626Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.487+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3777","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224487]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.487430424Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.487+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3778","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224487]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.487680146Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.487+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*vbt17w count=3779","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224487]},{"_index":"continuous-tests-pods-2023.11.09","_id":"up2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.487934263Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.487+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*vbt17w inflight=1 count=3780","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224487]},{"_index":"continuous-tests-pods-2023.11.09","_id":"u52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.487944236Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.487+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*vbt17w count=3781","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224487]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.488163726Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.488+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=46 cid=zb2*vbt17w bytes=65472 blockOffset=0 count=3782","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224488]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.488227143Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.488+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3783","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224488]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.488470165Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.488+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3784","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224488]},{"_index":"continuous-tests-pods-2023.11.09","_id":"v52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.488485646Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.488+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*iMDbvr count=3785","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224488]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.488676361Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.488+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*iMDbvr count=3786","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224488]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.488693021Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.488+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*iMDbvr peers=1 count=3787","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224488]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.488729457Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.488+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*iMDbvr inFlight=false count=3788","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224488]},{"_index":"continuous-tests-pods-2023.11.09","_id":"w52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.488822415Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.488+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*iMDbvr count=3789","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224488]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.488865351Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.488+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3790","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224488]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.488924344Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.488+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3791","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224488]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.488947985Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.488+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*iMDbvr count=3792","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224488]},{"_index":"continuous-tests-pods-2023.11.09","_id":"x52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.488996358Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.488+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*iMDbvr inFlight=true count=3793","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224488]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.489066800Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.488+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*iMDbvr count=3794","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224489]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.489107026Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.489+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3795","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224489]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.489144561Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.489+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3796","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224489]},{"_index":"continuous-tests-pods-2023.11.09","_id":"y52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.489192149Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.489+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3797","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224489]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zJ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.489429092Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.489+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*iMDbvr count=3798","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224489]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zZ2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.490199562Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.490+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3799","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224490]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zp2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.490429099Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.490+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*vbt17w inflight=0 count=3800","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224490]},{"_index":"continuous-tests-pods-2023.11.09","_id":"z52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.495555153Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.495+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3801","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224495]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0J2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.495645469Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.495+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3802","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224495]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0Z2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.496251089Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.496+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3803","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224496]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0p2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.496375021Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.496+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/vr/zb2rhmSzy55na4nEFVwHyambE95DHpTwhQWLUWAszDGiMDbvr + cid=zb2*iMDbvr count=3804","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224496]},{"_index":"continuous-tests-pods-2023.11.09","_id":"052EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.496752364Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.496+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=3193988 cid=zb2*iMDbvr count=3805","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224496]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1J2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.501943769Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.501+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3806","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224501]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1Z2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.502035808Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.501+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*iMDbvr count=3807","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224502]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1p2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.502045605Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.501+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=13286 count=3808","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224502]},{"_index":"continuous-tests-pods-2023.11.09","_id":"152EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.502050419Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.501+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3809","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224502]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2J2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.502107635Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.502+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*iMDbvr queue=0 count=3810","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224502]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2Z2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.502255620Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.502+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3811","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224502]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2p2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.502265789Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.502+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3812","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224502]},{"_index":"continuous-tests-pods-2023.11.09","_id":"252EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.502299600Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.502+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3813","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224502]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3J2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.502551847Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.502+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*iMDbvr count=3814","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224502]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3Z2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.502805410Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.502+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*iMDbvr inflight=1 count=3815","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224502]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3p2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.502847005Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.502+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*iMDbvr count=3816","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224502]},{"_index":"continuous-tests-pods-2023.11.09","_id":"352EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.503067161Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.502+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=47 cid=zb2*iMDbvr bytes=65472 blockOffset=0 count=3817","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224503]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4J2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.503108975Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.503+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3818","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224503]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4Z2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.503407940Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.503+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3819","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224503]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4p2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.503420785Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.503+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*fqA5zU count=3820","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224503]},{"_index":"continuous-tests-pods-2023.11.09","_id":"452EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.503546901Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.503+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*fqA5zU count=3821","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224503]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5J2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.503657832Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.503+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*fqA5zU peers=1 count=3822","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224503]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5Z2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.503695391Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.503+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*fqA5zU inFlight=false count=3823","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224503]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5p2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.503741336Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.503+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*fqA5zU count=3824","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224503]},{"_index":"continuous-tests-pods-2023.11.09","_id":"552EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.503777316Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.503+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3825","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224503]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6J2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.503784674Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.503+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3826","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224503]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6Z2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.503891971Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.503+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*fqA5zU count=3827","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224503]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6p2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.503938741Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.503+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*fqA5zU inFlight=true count=3828","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224503]},{"_index":"continuous-tests-pods-2023.11.09","_id":"652EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.503995362Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.503+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*fqA5zU count=3829","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224503]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7J2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.504035986Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.503+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3830","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224504]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7Z2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.504140596Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.503+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3831","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224504]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7p2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.504177397Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.504+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3832","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224504]},{"_index":"continuous-tests-pods-2023.11.09","_id":"752EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.504382252Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.504+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*fqA5zU count=3833","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224504]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8J2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.504737198Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.504+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3834","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224504]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8Z2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.504923435Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.504+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*iMDbvr inflight=0 count=3835","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224504]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8p2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.517500289Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.517+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3836","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224517]},{"_index":"continuous-tests-pods-2023.11.09","_id":"852EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.517560018Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.517+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3837","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224517]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9J2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.518132229Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.517+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3838","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224518]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9Z2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.518240123Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.518+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/zU/zb2rhhexbREttRwDCPpbx8XuLbyZZZpFZ6qwefNZnzBfqA5zU + cid=zb2*fqA5zU count=3839","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224518]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9p2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.518699339Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.518+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=3259460 cid=zb2*fqA5zU count=3840","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224518]},{"_index":"continuous-tests-pods-2023.11.09","_id":"952EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.524132895Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.523+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3841","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224524]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-J2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.524168904Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.523+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*fqA5zU count=3842","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224524]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-Z2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.524174829Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.523+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=20375 count=3843","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224524]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-p2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.524179540Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.524+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3844","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224524]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.524244971Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.524+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*fqA5zU queue=0 count=3845","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224524]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_J2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.524256097Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.524+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3846","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224524]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_Z2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.524260822Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.524+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3847","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224524]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_p2EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.524323602Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.524+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3848","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224524]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_52EtosBOvK4VyD1l7WL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.524495125Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.524+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*fqA5zU count=3849","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224524]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.524769843Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.524+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*fqA5zU inflight=1 count=3850","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224524]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.524785758Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.524+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*fqA5zU count=3851","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224524]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ap2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.525003204Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.524+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=48 cid=zb2*fqA5zU bytes=65472 blockOffset=0 count=3852","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224525]},{"_index":"continuous-tests-pods-2023.11.09","_id":"A52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.525018814Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.524+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3853","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224525]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.529332730Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.525+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3854","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224529]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.529376061Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.525+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*1SRErs count=3855","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224529]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Bp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.529381519Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.525+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*1SRErs count=3856","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224529]},{"_index":"continuous-tests-pods-2023.11.09","_id":"B52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.529386199Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.525+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*1SRErs peers=1 count=3857","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224529]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.529390613Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.525+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*1SRErs inFlight=false count=3858","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224529]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.529417185Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.525+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*1SRErs count=3859","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224529]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Cp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.529421774Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.525+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3860","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224529]},{"_index":"continuous-tests-pods-2023.11.09","_id":"C52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.529426955Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.525+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3861","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224529]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.529431326Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.525+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*1SRErs count=3862","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224529]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.529435672Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.525+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*1SRErs inFlight=true count=3863","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224529]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Dp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.529439934Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.525+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*1SRErs count=3864","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224529]},{"_index":"continuous-tests-pods-2023.11.09","_id":"D52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.529445434Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.525+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3865","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224529]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.529449862Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.525+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3866","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224529]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.529454459Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.526+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3867","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224529]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ep2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.529458805Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.526+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*1SRErs count=3868","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224529]},{"_index":"continuous-tests-pods-2023.11.09","_id":"E52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.529463255Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.526+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3869","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224529]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.529467557Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.527+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*fqA5zU inflight=0 count=3870","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224529]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.539602154Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.539+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3871","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224539]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Fp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.539691879Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.539+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3872","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224539]},{"_index":"continuous-tests-pods-2023.11.09","_id":"F52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.540735357Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.540+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3873","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224540]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.540828056Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.540+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/rs/zb2rhjNP9E2KoPB9TSUR2ayUSPMMPbz1k8nDzKZC9kb1SRErs + cid=zb2*1SRErs count=3874","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224540]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.541345116Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.541+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=3324932 cid=zb2*1SRErs count=3875","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224541]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Gp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.547197221Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.546+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3876","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224547]},{"_index":"continuous-tests-pods-2023.11.09","_id":"G52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.547235189Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.546+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*1SRErs count=3877","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224547]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.547249146Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.547+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=21441 count=3878","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224547]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.547335120Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.547+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3879","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224547]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Hp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.547345383Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.547+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*1SRErs queue=0 count=3880","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224547]},{"_index":"continuous-tests-pods-2023.11.09","_id":"H52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.547404061Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.547+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3881","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224547]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.547409229Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.547+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3882","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224547]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.547436904Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.547+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3883","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224547]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ip2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.547742371Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.547+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*1SRErs count=3884","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224547]},{"_index":"continuous-tests-pods-2023.11.09","_id":"I52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.547898826Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.547+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*1SRErs inflight=1 count=3885","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224547]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.547910719Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.547+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*1SRErs count=3886","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224547]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.548201263Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.548+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=49 cid=zb2*1SRErs bytes=65472 blockOffset=0 count=3887","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224548]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Jp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.548234061Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.548+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3888","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224548]},{"_index":"continuous-tests-pods-2023.11.09","_id":"J52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.548540228Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.548+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3889","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224548]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.548554166Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.548+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*9aFnte count=3890","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224548]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.548761116Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.548+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*9aFnte count=3891","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224548]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Kp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.548777736Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.548+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*9aFnte peers=1 count=3892","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224548]},{"_index":"continuous-tests-pods-2023.11.09","_id":"K52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.548799398Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.548+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*9aFnte inFlight=false count=3893","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224548]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.548937009Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.548+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*9aFnte count=3894","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224548]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.548952264Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.548+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3895","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224548]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Lp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.548962922Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.548+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3896","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224548]},{"_index":"continuous-tests-pods-2023.11.09","_id":"L52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.548967486Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.548+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*9aFnte count=3897","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224548]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.549037161Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.548+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*9aFnte inFlight=true count=3898","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224549]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.549136401Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.548+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*9aFnte count=3899","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224549]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Mp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.549148011Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.549+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3900","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224549]},{"_index":"continuous-tests-pods-2023.11.09","_id":"M52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.549192645Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.549+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3901","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224549]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.549252830Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.549+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3902","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224549]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.549501462Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.549+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*9aFnte count=3903","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224549]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Np2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.550425927Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.550+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3904","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224550]},{"_index":"continuous-tests-pods-2023.11.09","_id":"N52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.550565549Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.550+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*1SRErs inflight=0 count=3905","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224550]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.559491304Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.558+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3906","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224559]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.559534949Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.559+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3907","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224559]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Op2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.560317320Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.560+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3908","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224560]},{"_index":"continuous-tests-pods-2023.11.09","_id":"O52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.560467613Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.560+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/te/zb2rhdf1eBACFWXriC9nkqDjw6gtBNRbWpor9KoVuVX9aFnte + cid=zb2*9aFnte count=3909","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224560]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.560880068Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.560+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=3390404 cid=zb2*9aFnte count=3910","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224560]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.566691439Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.566+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3911","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224566]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Pp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.566733183Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.566+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*9aFnte count=3912","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224566]},{"_index":"continuous-tests-pods-2023.11.09","_id":"P52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.566739175Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.566+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=17900 count=3913","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224566]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.566743350Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.566+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3914","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224566]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.566957939Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.566+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*9aFnte queue=0 count=3915","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224566]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Qp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.566974368Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.566+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3916","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224566]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Q52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.566979419Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.566+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3917","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224566]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.566984446Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.566+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3918","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224566]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.567212377Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.567+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*9aFnte count=3919","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224567]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Rp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.567407849Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.567+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*9aFnte inflight=1 count=3920","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224567]},{"_index":"continuous-tests-pods-2023.11.09","_id":"R52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.567448172Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.567+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*9aFnte count=3921","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224567]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.567629125Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.567+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=50 cid=zb2*9aFnte bytes=65472 blockOffset=0 count=3922","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224567]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.567650215Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.567+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3923","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224567]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Sp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.567979361Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.567+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3924","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224567]},{"_index":"continuous-tests-pods-2023.11.09","_id":"S52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.567996286Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.567+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*PsoM11 count=3925","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224567]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.568115364Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.568+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*PsoM11 count=3926","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224568]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.568188622Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.568+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*PsoM11 peers=1 count=3927","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224568]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Tp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.568231979Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.568+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*PsoM11 inFlight=false count=3928","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224568]},{"_index":"continuous-tests-pods-2023.11.09","_id":"T52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.568271379Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.568+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*PsoM11 count=3929","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224568]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.568346402Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.568+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3930","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224568]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.568366495Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.568+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3931","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224568]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Up2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.568418456Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.568+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*PsoM11 count=3932","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224568]},{"_index":"continuous-tests-pods-2023.11.09","_id":"U52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.568432355Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.568+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*PsoM11 inFlight=true count=3933","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224568]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.568495179Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.568+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*PsoM11 count=3934","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224568]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.568550321Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.568+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3935","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224568]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Vp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.568665830Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.568+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3936","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224568]},{"_index":"continuous-tests-pods-2023.11.09","_id":"V52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.568678271Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.568+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3937","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224568]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.568865866Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.568+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*PsoM11 count=3938","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224568]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.583566764Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.583+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3939","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224583]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Wp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.583745596Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.583+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*9aFnte inflight=0 count=3940","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224583]},{"_index":"continuous-tests-pods-2023.11.09","_id":"W52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.585588472Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.585+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3941","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224585]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.585624316Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.585+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3942","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224585]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.586497993Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.586+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3943","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224586]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Xp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.586631931Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.586+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/11/zb2rhkP8prgXuMC6sZLyVsyPHdxqRpwX8mWpdYUqzLGPsoM11 + cid=zb2*PsoM11 count=3944","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224586]},{"_index":"continuous-tests-pods-2023.11.09","_id":"X52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.587242946Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.587+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=3455876 cid=zb2*PsoM11 count=3945","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224587]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.593285425Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.592+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3946","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224593]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.593319836Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.593+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*PsoM11 count=3947","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224593]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Yp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.593343626Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.593+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=24973 count=3948","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224593]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Y52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.593348042Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.593+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3949","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224593]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.593359823Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.593+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*PsoM11 queue=0 count=3950","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224593]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.593438152Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.593+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3951","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224593]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Zp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.593504282Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.593+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3952","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224593]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Z52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.593529799Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.593+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3953","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224593]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.593661974Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.593+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*PsoM11 count=3954","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224593]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.593941329Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.593+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*PsoM11 inflight=1 count=3955","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224593]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ap2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.593955904Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.593+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*PsoM11 count=3956","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224593]},{"_index":"continuous-tests-pods-2023.11.09","_id":"a52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.594113649Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.593+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=51 cid=zb2*PsoM11 bytes=65472 blockOffset=0 count=3957","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224594]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.594130631Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.594+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3958","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224594]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.594642679Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.594+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3959","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224594]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.594662588Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.594+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*wBrUqJ count=3960","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224594]},{"_index":"continuous-tests-pods-2023.11.09","_id":"b52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.594678245Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.594+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*wBrUqJ count=3961","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224594]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.594845879Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.594+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*wBrUqJ peers=1 count=3962","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224594]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.594864630Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.594+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*wBrUqJ inFlight=false count=3963","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224594]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.594899828Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.594+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*wBrUqJ count=3964","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224594]},{"_index":"continuous-tests-pods-2023.11.09","_id":"c52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.594994713Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.594+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=3965","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224594]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.595010870Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.594+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=3966","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224595]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.595057223Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.594+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*wBrUqJ count=3967","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224595]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.595087492Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.595+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*wBrUqJ inFlight=true count=3968","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224595]},{"_index":"continuous-tests-pods-2023.11.09","_id":"d52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.595121243Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.595+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*wBrUqJ count=3969","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224595]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.595197772Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.595+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=3970","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224595]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.595286922Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.595+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=3971","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224595]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ep2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.595294571Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.595+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3972","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224595]},{"_index":"continuous-tests-pods-2023.11.09","_id":"e52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.595574153Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.595+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*wBrUqJ count=3973","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224595]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.600931406Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.600+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=3974","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224600]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.601149753Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.600+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*PsoM11 inflight=0 count=3975","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224601]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.608250237Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.607+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=3976","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224608]},{"_index":"continuous-tests-pods-2023.11.09","_id":"f52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.608285031Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.608+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=3977","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224608]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.609075130Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.608+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=3978","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224609]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.609248594Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.609+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/qJ/zb2rhdF3XMXz3vXKTiPkqkRaPWC45KAktKtCa8cPNHswBrUqJ + cid=zb2*wBrUqJ count=3979","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224609]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.609846918Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.609+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=3521348 cid=zb2*wBrUqJ count=3980","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224609]},{"_index":"continuous-tests-pods-2023.11.09","_id":"g52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.619569842Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.619+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3981","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224619]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.619604563Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.619+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*wBrUqJ count=3982","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224619]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.619611146Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.619+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=24630 count=3983","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224619]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.619616547Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.619+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=3984","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224619]},{"_index":"continuous-tests-pods-2023.11.09","_id":"h52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.619676126Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.619+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*wBrUqJ queue=0 count=3985","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224619]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.619683825Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.619+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=3986","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224619]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.619697520Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.619+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=3987","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224619]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ip2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.619783124Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.619+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=3988","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224619]},{"_index":"continuous-tests-pods-2023.11.09","_id":"i52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.619907334Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.619+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*wBrUqJ count=3989","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224619]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.620205510Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.620+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*wBrUqJ inflight=1 count=3990","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224620]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.620222158Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.620+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*wBrUqJ count=3991","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224620]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.620467817Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.620+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=52 cid=zb2*wBrUqJ bytes=65472 blockOffset=0 count=3992","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224620]},{"_index":"continuous-tests-pods-2023.11.09","_id":"j52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.620490514Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.620+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=3993","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224620]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.620794870Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.620+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=3994","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224620]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.620811676Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.620+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*3rVX4V count=3995","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224620]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.620914771Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.620+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*3rVX4V count=3996","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224620]},{"_index":"continuous-tests-pods-2023.11.09","_id":"k52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.621013022Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.620+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*3rVX4V peers=1 count=3997","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224621]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.621028636Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.620+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*3rVX4V inFlight=false count=3998","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224621]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.621064355Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.621+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*3rVX4V count=3999","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224621]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.621130846Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.621+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4000","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224621]},{"_index":"continuous-tests-pods-2023.11.09","_id":"l52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.621142917Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.621+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4001","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224621]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.621230930Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.621+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*3rVX4V count=4002","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224621]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.621265417Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.621+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*3rVX4V inFlight=true count=4003","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224621]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.621276428Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.621+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*3rVX4V count=4004","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224621]},{"_index":"continuous-tests-pods-2023.11.09","_id":"m52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.621347831Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.621+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4005","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224621]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.621428892Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.621+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4006","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224621]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.621458287Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.621+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4007","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224621]},{"_index":"continuous-tests-pods-2023.11.09","_id":"np2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.621651689Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.621+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*3rVX4V count=4008","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224621]},{"_index":"continuous-tests-pods-2023.11.09","_id":"n52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.622283354Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.622+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4009","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224622]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.622501652Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.622+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*wBrUqJ inflight=0 count=4010","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224622]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.627454237Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.627+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4011","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224627]},{"_index":"continuous-tests-pods-2023.11.09","_id":"op2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.627511046Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.627+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4012","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224627]},{"_index":"continuous-tests-pods-2023.11.09","_id":"o52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.628271383Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.628+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4013","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224628]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.628495864Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.628+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/4V/zb2rhoryKkgxAXjbrGMNvzTWZxQFsjqGDsxrhAAyCG43rVX4V + cid=zb2*3rVX4V count=4014","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224628]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.629332208Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.628+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=3586820 cid=zb2*3rVX4V count=4015","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224629]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.634812448Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.634+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4016","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224634]},{"_index":"continuous-tests-pods-2023.11.09","_id":"p52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.634842489Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.634+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*3rVX4V count=4017","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224634]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.634847900Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.634+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=13709 count=4018","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224634]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.635003632Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.634+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4019","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224635]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.635013023Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.634+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*3rVX4V queue=0 count=4020","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224635]},{"_index":"continuous-tests-pods-2023.11.09","_id":"q52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.635017275Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.634+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4021","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224635]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.635053893Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.634+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4022","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224635]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.635087137Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.635+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4023","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224635]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.635349236Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.635+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*3rVX4V count=4024","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224635]},{"_index":"continuous-tests-pods-2023.11.09","_id":"r52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.635496290Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.635+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*3rVX4V inflight=1 count=4025","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224635]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.635508560Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.635+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*3rVX4V count=4026","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224635]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.635808937Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.635+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=53 cid=zb2*3rVX4V bytes=65472 blockOffset=0 count=4027","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224635]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.635824863Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.635+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4028","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224635]},{"_index":"continuous-tests-pods-2023.11.09","_id":"s52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.636126352Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.635+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4029","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224636]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.636144622Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.635+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*fUx1pi count=4030","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224636]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.636277282Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.636+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*fUx1pi count=4031","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224636]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.636332070Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.636+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*fUx1pi peers=1 count=4032","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224636]},{"_index":"continuous-tests-pods-2023.11.09","_id":"t52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.636344243Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.636+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*fUx1pi inFlight=false count=4033","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224636]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.636443190Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.636+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*fUx1pi count=4034","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224636]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.636479059Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.636+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4035","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224636]},{"_index":"continuous-tests-pods-2023.11.09","_id":"up2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.636491237Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.636+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4036","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224636]},{"_index":"continuous-tests-pods-2023.11.09","_id":"u52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.636516284Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.636+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*fUx1pi count=4037","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224636]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.636576797Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.636+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*fUx1pi inFlight=true count=4038","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224636]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.636864397Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.636+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*fUx1pi count=4039","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224636]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.636883307Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.636+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4040","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224636]},{"_index":"continuous-tests-pods-2023.11.09","_id":"v52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.636888866Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.636+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4041","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224636]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.636893478Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.636+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4042","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224636]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.637003700Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.636+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*fUx1pi count=4043","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224637]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.637706053Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.637+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4044","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224637]},{"_index":"continuous-tests-pods-2023.11.09","_id":"w52EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.637857076Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.637+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*3rVX4V inflight=0 count=4045","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224637]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xJ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.642929450Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.642+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4046","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224642]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xZ2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.642976261Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.642+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4047","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224642]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xp2EtosBOvK4VyD1l7aL","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.643673265Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.643+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4048","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224643]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4p2EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.643811367Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.643+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/pi/zb2rhXuaNZy3t32PZ74TrwD77mViV6YwWZy1ByyKuWefUx1pi + cid=zb2*fUx1pi count=4049","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224643]},{"_index":"continuous-tests-pods-2023.11.09","_id":"452EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.644280259Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.644+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=3652292 cid=zb2*fUx1pi count=4050","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224644]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5J2EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.649896024Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.649+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4051","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224649]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5Z2EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.649932009Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.649+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*fUx1pi count=4052","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224649]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5p2EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.649947504Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.649+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=13569 count=4053","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224649]},{"_index":"continuous-tests-pods-2023.11.09","_id":"552EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.649952774Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.649+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4054","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224649]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6J2EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.649957888Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.649+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*fUx1pi queue=0 count=4055","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224649]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7Z2EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.650576749Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.650+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*fUx1pi inflight=1 count=4060","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224650]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7p2EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.650594367Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.650+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*fUx1pi count=4061","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224650]},{"_index":"continuous-tests-pods-2023.11.09","_id":"752EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.650831864Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.650+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=54 cid=zb2*fUx1pi bytes=65472 blockOffset=0 count=4062","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224650]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6Z2EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.650138990Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.649+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4056","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224650]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6p2EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.650163469Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.650+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4057","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224650]},{"_index":"continuous-tests-pods-2023.11.09","_id":"652EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.650169811Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.650+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4058","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224650]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7J2EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.650301873Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.650+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*fUx1pi count=4059","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224650]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8J2EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.650845387Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.650+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4063","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224650]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_Z2EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.651857576Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.651+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4076","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224651]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_p2EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.651865146Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.651+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4077","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224651]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8Z2EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.651172276Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.650+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4064","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224651]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8p2EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.651188192Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.651+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*g8DbDE count=4065","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224651]},{"_index":"continuous-tests-pods-2023.11.09","_id":"852EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.651308871Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.651+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*g8DbDE count=4066","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224651]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9J2EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.651420309Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.651+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*g8DbDE peers=1 count=4067","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224651]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9Z2EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.651430508Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.651+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*g8DbDE inFlight=false count=4068","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224651]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9p2EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.651483211Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.651+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*g8DbDE count=4069","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224651]},{"_index":"continuous-tests-pods-2023.11.09","_id":"952EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.651528635Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.651+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4070","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224651]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-J2EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.651534905Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.651+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4071","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224651]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-Z2EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.651671910Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.651+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*g8DbDE count=4072","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224651]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-p2EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.651680037Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.651+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*g8DbDE inFlight=true count=4073","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224651]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-52EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.651688839Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.651+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*g8DbDE count=4074","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224651]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_J2EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.651736414Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.651+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4075","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224651]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.652539241Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.652+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*fUx1pi inflight=0 count=4080","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224652]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_52EtosBOvK4VyD1mLkt","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.652082069Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.651+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*g8DbDE count=4078","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224652]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.652404504Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.652+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4079","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224652]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ap2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.664205931Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.663+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4081","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224664]},{"_index":"continuous-tests-pods-2023.11.09","_id":"A52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.664266757Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.663+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4082","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224664]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.664810476Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.664+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4083","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224664]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.664939842Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.664+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/DE/zb2rhbEZuvoA5rZSVkAMQXUP1U3QLpUjP4b6yUyH7Etg8DbDE + cid=zb2*g8DbDE count=4084","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224664]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Bp2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.665487121Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.665+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=3717764 cid=zb2*g8DbDE count=4085","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224665]},{"_index":"continuous-tests-pods-2023.11.09","_id":"B52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.671458545Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.671+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4086","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224671]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.671494884Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.671+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*g8DbDE count=4087","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224671]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.671501509Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.671+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=19896 count=4088","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224671]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Cp2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.671506158Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.671+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4089","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224671]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Dp2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.671581134Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.671+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4093","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224671]},{"_index":"continuous-tests-pods-2023.11.09","_id":"C52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.671510899Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.671+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*g8DbDE queue=0 count=4090","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224671]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.671527198Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.671+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4091","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224671]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.671556841Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.671+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4092","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224671]},{"_index":"continuous-tests-pods-2023.11.09","_id":"D52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.671796860Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.671+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*g8DbDE count=4094","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224671]},{"_index":"continuous-tests-pods-2023.11.09","_id":"G52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.672998309Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.672+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4106","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224672]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.672671723Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.672+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*y6mFnR count=4100","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224672]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Fp2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.672822358Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.672+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*y6mFnR count=4101","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224672]},{"_index":"continuous-tests-pods-2023.11.09","_id":"F52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.672861046Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.672+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*y6mFnR peers=1 count=4102","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224672]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.672884865Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.672+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*y6mFnR inFlight=false count=4103","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224672]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.672975018Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.672+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*y6mFnR count=4104","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224672]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Gp2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.672992472Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.672+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4105","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224672]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.672157860Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.671+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*g8DbDE inflight=1 count=4095","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224672]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.672171486Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.671+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*g8DbDE count=4096","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224672]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ep2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.672224299Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.672+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=55 cid=zb2*g8DbDE bytes=65472 blockOffset=0 count=4097","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224672]},{"_index":"continuous-tests-pods-2023.11.09","_id":"E52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.672317805Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.672+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4098","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224672]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.672659066Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.672+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4099","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224672]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.673100957Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.672+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*y6mFnR count=4107","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224673]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.673140067Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.673+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*y6mFnR inFlight=true count=4108","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224673]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Hp2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.673151454Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.673+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*y6mFnR count=4109","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224673]},{"_index":"continuous-tests-pods-2023.11.09","_id":"H52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.673190741Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.673+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4110","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224673]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.673385110Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.673+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4112","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224673]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.673901652Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.673+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*g8DbDE inflight=0 count=4115","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224673]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.673370370Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.673+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4111","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224673]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ip2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.673511987Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.673+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*y6mFnR count=4113","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224673]},{"_index":"continuous-tests-pods-2023.11.09","_id":"I52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.673746313Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.673+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4114","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224673]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.687403044Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.686+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4116","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224687]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Jp2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.687446355Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.687+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4117","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224687]},{"_index":"continuous-tests-pods-2023.11.09","_id":"J52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.688013231Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.687+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4118","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224688]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.688223222Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.687+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/nR/zb2rhbJWz4oowpixEiux7CrJuSNFDvCrPdoWCi1c26Sy6mFnR + cid=zb2*y6mFnR count=4119","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224688]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.688704514Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.688+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=3783236 cid=zb2*y6mFnR count=4120","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224688]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Kp2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.694859451Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.694+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4121","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224694]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.694954033Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.694+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4124","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224694]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Lp2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.694958955Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.694+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*y6mFnR queue=0 count=4125","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224694]},{"_index":"continuous-tests-pods-2023.11.09","_id":"K52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.694934264Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.694+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*y6mFnR count=4122","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224694]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.694941081Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.694+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=21909 count=4123","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224694]},{"_index":"continuous-tests-pods-2023.11.09","_id":"L52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.695080761Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.694+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4126","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224695]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.695105187Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.694+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4127","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224695]},{"_index":"continuous-tests-pods-2023.11.09","_id":"N52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.695879093Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.695+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4134","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224695]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.695890676Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.695+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*MEF2ur count=4135","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224695]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.695110872Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.695+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4128","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224695]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Mp2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.695244717Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.695+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*y6mFnR count=4129","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224695]},{"_index":"continuous-tests-pods-2023.11.09","_id":"M52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.695430415Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.695+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*y6mFnR inflight=1 count=4130","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224695]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.695459851Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.695+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*y6mFnR count=4131","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224695]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.695613813Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.695+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=56 cid=zb2*y6mFnR bytes=65472 blockOffset=0 count=4132","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224695]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Np2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.695633606Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.695+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4133","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224695]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.696028748Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.695+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*MEF2ur count=4136","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224696]},{"_index":"continuous-tests-pods-2023.11.09","_id":"O52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.696132437Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.696+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*MEF2ur inFlight=false count=4138","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224696]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.696168213Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.696+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*MEF2ur count=4139","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224696]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.696205472Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.696+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4140","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224696]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Pp2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.696212214Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.696+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4141","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224696]},{"_index":"continuous-tests-pods-2023.11.09","_id":"P52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.696340825Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.696+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*MEF2ur count=4142","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224696]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Qp2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.696444384Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.696+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4145","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224696]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Q52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.696574561Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.696+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4146","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224696]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.696594225Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.696+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4147","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224696]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.696788970Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.696+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*MEF2ur count=4148","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224696]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Op2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.696095561Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.695+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*MEF2ur peers=1 count=4137","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224696]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.696353262Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.696+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*MEF2ur inFlight=true count=4143","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224696]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.696425475Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.696+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*MEF2ur count=4144","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224696]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Rp2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.697768989Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.697+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4149","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224697]},{"_index":"continuous-tests-pods-2023.11.09","_id":"R52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.697860378Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.697+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*y6mFnR inflight=0 count=4150","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224697]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.710086100Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.709+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4151","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224710]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.710124852Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.709+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4152","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224710]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Sp2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.710602260Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.710+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4153","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224710]},{"_index":"continuous-tests-pods-2023.11.09","_id":"S52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.710816826Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.710+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/ur/zb2rhXVwkEfxGj1QchHT6TU3c2pX9gmVTFxQfUXX8zyMEF2ur + cid=zb2*MEF2ur count=4154","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224710]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.711484182Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.711+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=3848708 cid=zb2*MEF2ur count=4155","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224711]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.719932619Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.719+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4156","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224719]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Tp2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.719962480Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.719+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*MEF2ur count=4157","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224719]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.719975241Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.719+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*MEF2ur queue=0 count=4160","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224719]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Up2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.719997333Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.719+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4161","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224719]},{"_index":"continuous-tests-pods-2023.11.09","_id":"T52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.719967345Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.719+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=23607 count=4158","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224719]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.719971411Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.719+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4159","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224719]},{"_index":"continuous-tests-pods-2023.11.09","_id":"U52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.720018237Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.719+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4162","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224720]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.720030611Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.719+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4163","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224720]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.720223081Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.720+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*MEF2ur count=4164","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224720]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Vp2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.720346638Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.720+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*MEF2ur inflight=1 count=4165","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224720]},{"_index":"continuous-tests-pods-2023.11.09","_id":"V52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.720367739Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.720+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*MEF2ur count=4166","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224720]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.720601830Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.720+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=57 cid=zb2*MEF2ur bytes=65472 blockOffset=0 count=4167","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224720]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.720627039Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.720+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4168","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224720]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Wp2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.720796571Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.720+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4169","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224720]},{"_index":"continuous-tests-pods-2023.11.09","_id":"W52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.720842198Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.720+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*1CsoUw count=4170","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224720]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.721005056Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.720+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*1CsoUw count=4171","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224721]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Xp2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.721038766Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.720+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*1CsoUw inFlight=false count=4173","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224721]},{"_index":"continuous-tests-pods-2023.11.09","_id":"X52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.721118359Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.721+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*1CsoUw count=4174","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224721]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.721143755Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.721+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4176","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224721]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Yp2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.721192363Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.721+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*1CsoUw count=4177","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224721]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Y52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.721228923Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.721+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*1CsoUw inFlight=true count=4178","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224721]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.721325728Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.721+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*1CsoUw count=4179","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224721]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.721342895Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.721+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4180","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224721]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Zp2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.721403546Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.721+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4181","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224721]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Z52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.721419881Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.721+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4182","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224721]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.721624560Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.721+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*1CsoUw count=4183","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224721]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.721021762Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.720+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*1CsoUw peers=1 count=4172","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224721]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.721124261Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.721+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4175","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224721]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ap2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.722470016Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.722+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*MEF2ur inflight=0 count=4185","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224722]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.722294939Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.722+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4184","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224722]},{"_index":"continuous-tests-pods-2023.11.09","_id":"a52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.734486140Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.734+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4186","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224734]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.734523832Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.734+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4187","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224734]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.735305834Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.734+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4188","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224735]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bp2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.735344012Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.735+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/Uw/zb2rhjhqeEqMGPNF17AYdLbCyhfTFAAbXMnhpbF8Uck1CsoUw + cid=zb2*1CsoUw count=4189","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224735]},{"_index":"continuous-tests-pods-2023.11.09","_id":"b52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.735752087Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.735+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=3914180 cid=zb2*1CsoUw count=4190","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224735]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.741661970Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.741+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4191","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224741]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.741724041Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.741+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*1CsoUw count=4192","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224741]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cp2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.741730673Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.741+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=20533 count=4193","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224741]},{"_index":"continuous-tests-pods-2023.11.09","_id":"c52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.741745685Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.741+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4194","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224741]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.741767144Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.741+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*1CsoUw queue=0 count=4195","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224741]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.741777200Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.741+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4196","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224741]},{"_index":"continuous-tests-pods-2023.11.09","_id":"d52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.741904512Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.741+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4198","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224741]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dp2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.741781484Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.741+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4197","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224741]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fp2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.742704676Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.742+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*dU49gs count=4205","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224742]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.742922780Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.742+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*dU49gs peers=1 count=4207","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224742]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.742939086Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.742+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*dU49gs inFlight=false count=4208","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224742]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.742026169Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.741+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*1CsoUw count=4199","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224742]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.742171938Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.742+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*1CsoUw inflight=1 count=4200","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224742]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ep2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.742184136Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.742+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*1CsoUw count=4201","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224742]},{"_index":"continuous-tests-pods-2023.11.09","_id":"e52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.742472119Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.742+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=58 cid=zb2*1CsoUw bytes=65472 blockOffset=0 count=4202","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224742]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fJ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.742507947Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.742+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4203","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224742]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fZ2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.742695261Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.742+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4204","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224742]},{"_index":"continuous-tests-pods-2023.11.09","_id":"f52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.742911498Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.742+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*dU49gs count=4206","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224742]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gp2EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.743043804Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.742+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*dU49gs count=4209","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224743]},{"_index":"continuous-tests-pods-2023.11.09","_id":"g52EtosBOvK4VyD1mLot","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.743064498Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.743+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4210","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224743]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hJ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.743082629Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.743+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4211","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224743]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hZ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.743103059Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.743+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*dU49gs count=4212","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224743]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hp2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.743154226Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.743+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*dU49gs inFlight=true count=4213","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224743]},{"_index":"continuous-tests-pods-2023.11.09","_id":"h52EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.743216682Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.743+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*dU49gs count=4214","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224743]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iJ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.743253451Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.743+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4215","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224743]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iZ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.743288430Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.743+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4216","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224743]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ip2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.743330406Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.743+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4217","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224743]},{"_index":"continuous-tests-pods-2023.11.09","_id":"i52EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.743520999Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.743+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*dU49gs count=4218","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224743]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jJ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.744115058Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.743+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4219","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224744]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jZ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.744200822Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.744+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*1CsoUw inflight=0 count=4220","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224744]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jp2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.755400949Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.755+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4221","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224755]},{"_index":"continuous-tests-pods-2023.11.09","_id":"j52EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.755448838Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.755+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4222","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224755]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kJ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.756176699Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.755+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4223","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224756]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kZ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.756216430Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.756+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/gs/zb2rhjvjDfWtq9ihPyhs5rHvfwBZgTkDCDTzWnGGnGbdU49gs + cid=zb2*dU49gs count=4224","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224756]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kp2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.756715100Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.756+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=3979652 cid=zb2*dU49gs count=4225","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224756]},{"_index":"continuous-tests-pods-2023.11.09","_id":"l52EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.762136398Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.762+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*dU49gs queue=0 count=4230","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224762]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nZ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.762641199Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.762+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*dU49gs count=4236","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224762]},{"_index":"continuous-tests-pods-2023.11.09","_id":"k52EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.762043586Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.761+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4226","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224762]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lJ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.762079879Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.761+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*dU49gs count=4227","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224762]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lZ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.762105251Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.761+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=19035 count=4228","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224762]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lp2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.762126445Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.762+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4229","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224762]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mJ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.762170980Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.762+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4231","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224762]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mZ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.762180670Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.762+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4232","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224762]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mp2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.762306296Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.762+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4233","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224762]},{"_index":"continuous-tests-pods-2023.11.09","_id":"m52EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.762477953Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.762+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*dU49gs count=4234","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224762]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nJ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.762630961Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.762+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*dU49gs inflight=1 count=4235","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224762]},{"_index":"continuous-tests-pods-2023.11.09","_id":"np2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.762947367Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.762+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=59 cid=zb2*dU49gs bytes=65472 blockOffset=0 count=4237","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224762]},{"_index":"continuous-tests-pods-2023.11.09","_id":"n52EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.762986185Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.762+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4238","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224762]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qZ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.763604788Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.763+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*Vh5JZS inFlight=true count=4248","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224763]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oJ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.763174534Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.763+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4239","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224763]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oZ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.763183702Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.763+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*Vh5JZS count=4240","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224763]},{"_index":"continuous-tests-pods-2023.11.09","_id":"op2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.763310869Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.763+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*Vh5JZS count=4241","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224763]},{"_index":"continuous-tests-pods-2023.11.09","_id":"o52EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.763345526Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.763+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*Vh5JZS peers=1 count=4242","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224763]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pJ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.763441008Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.763+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*Vh5JZS inFlight=false count=4243","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224763]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pZ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.763471680Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.763+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*Vh5JZS count=4244","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224763]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pp2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.763478759Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.763+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4245","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224763]},{"_index":"continuous-tests-pods-2023.11.09","_id":"p52EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.763487188Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.763+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4246","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224763]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qJ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.763551159Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.763+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*Vh5JZS count=4247","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224763]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qp2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.763623908Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.763+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*Vh5JZS count=4249","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224763]},{"_index":"continuous-tests-pods-2023.11.09","_id":"q52EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.763660072Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.763+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4250","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224763]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rJ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.763694942Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.763+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4251","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224763]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rZ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.763739225Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.763+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4252","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224763]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rp2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.763988194Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.763+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*Vh5JZS count=4253","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224763]},{"_index":"continuous-tests-pods-2023.11.09","_id":"r52EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.764527268Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.764+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4254","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224764]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sJ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.764547529Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.764+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*dU49gs inflight=0 count=4255","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224764]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sZ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.777939288Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.777+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4256","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224777]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sp2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.777976045Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.777+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4257","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224777]},{"_index":"continuous-tests-pods-2023.11.09","_id":"s52EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.778747059Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.778+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4258","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224778]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tJ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.778773772Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.778+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/ZS/zb2rhfXEDgpiEYL6ASi3iN4NmbyM9VUKD3QVcvfwEvcVh5JZS + cid=zb2*Vh5JZS count=4259","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224778]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tZ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.779278437Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.779+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=4045124 cid=zb2*Vh5JZS count=4260","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224779]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uZ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.785069694Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.784+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4264","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224785]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vZ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.785189145Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.785+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4268","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224785]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vp2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.785371716Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.785+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*Vh5JZS count=4269","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224785]},{"_index":"continuous-tests-pods-2023.11.09","_id":"v52EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.785643025Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.785+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*Vh5JZS inflight=1 count=4270","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224785]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wJ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.785659578Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.785+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*Vh5JZS count=4271","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224785]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tp2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.785021816Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.784+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4261","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224785]},{"_index":"continuous-tests-pods-2023.11.09","_id":"t52EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.785059225Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.784+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*Vh5JZS count=4262","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224785]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uJ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.785064922Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.784+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=21533 count=4263","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224785]},{"_index":"continuous-tests-pods-2023.11.09","_id":"up2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.785081485Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.784+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*Vh5JZS queue=0 count=4265","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224785]},{"_index":"continuous-tests-pods-2023.11.09","_id":"u52EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.785113010Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.785+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4266","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224785]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vJ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.785141359Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.785+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4267","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224785]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wZ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.785864651Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.785+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=60 cid=zb2*Vh5JZS bytes=65472 blockOffset=0 count=4272","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224785]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wp2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.785887568Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.785+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4273","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224785]},{"_index":"continuous-tests-pods-2023.11.09","_id":"x52EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.786551471Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.786+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*JrTkBJ inFlight=false count=4278","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224786]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yJ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.786627782Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.786+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*JrTkBJ count=4279","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224786]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yZ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.786634742Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.786+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4280","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224786]},{"_index":"continuous-tests-pods-2023.11.09","_id":"w52EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.786173654Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.786+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4274","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224786]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xJ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.786198595Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.786+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*JrTkBJ count=4275","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224786]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xZ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.786329116Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.786+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*JrTkBJ count=4276","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224786]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xp2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.786515819Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.786+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*JrTkBJ peers=1 count=4277","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224786]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yp2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.786722870Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.786+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4281","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224786]},{"_index":"continuous-tests-pods-2023.11.09","_id":"y52EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.786750215Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.786+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*JrTkBJ count=4282","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224786]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zJ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.786760010Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.786+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*JrTkBJ inFlight=true count=4283","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224786]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zZ2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.786809189Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.786+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*JrTkBJ count=4284","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224786]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zp2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.786833764Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.786+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4285","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224786]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0Z2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.787312850Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.787+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*JrTkBJ count=4288","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224787]},{"_index":"continuous-tests-pods-2023.11.09","_id":"z52EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.787015743Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.786+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4286","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224787]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0J2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.787032363Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.786+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4287","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224787]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0p2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.788661709Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.788+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4289","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224788]},{"_index":"continuous-tests-pods-2023.11.09","_id":"052EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.788696743Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.788+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*Vh5JZS inflight=0 count=4290","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224788]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1J2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.793197217Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.792+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4291","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224793]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1Z2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.793231299Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.793+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4292","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224793]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1p2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.793773098Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.793+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4293","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224793]},{"_index":"continuous-tests-pods-2023.11.09","_id":"152EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.793880780Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.793+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/BJ/zb2rhanwyQFsX9QMcjusKW39LnUudRL8v1yqwrZM27cJrTkBJ + cid=zb2*JrTkBJ count=4294","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224793]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2J2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.794475916Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.794+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=4110596 cid=zb2*JrTkBJ count=4295","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224794]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2Z2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.800455622Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.800+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4296","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224800]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2p2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.800485335Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.800+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*JrTkBJ count=4297","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224800]},{"_index":"continuous-tests-pods-2023.11.09","_id":"252EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.800490857Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.800+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=13808 count=4298","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224800]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3J2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.800494649Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.800+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4299","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224800]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3Z2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.800525977Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.800+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*JrTkBJ queue=0 count=4300","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224800]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3p2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.800555933Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.800+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4301","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224800]},{"_index":"continuous-tests-pods-2023.11.09","_id":"352EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.800561159Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.800+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4302","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224800]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4J2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.800599707Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.800+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4303","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224800]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4Z2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.800767436Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.800+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*JrTkBJ count=4304","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224800]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4p2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.801044866Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.800+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*JrTkBJ inflight=1 count=4305","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224801]},{"_index":"continuous-tests-pods-2023.11.09","_id":"452EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.801053168Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.800+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*JrTkBJ count=4306","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224801]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5J2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.801236720Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.801+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=61 cid=zb2*JrTkBJ bytes=65472 blockOffset=0 count=4307","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224801]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5Z2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.801279619Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.801+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4308","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224801]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5p2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.801635747Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.801+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4309","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224801]},{"_index":"continuous-tests-pods-2023.11.09","_id":"552EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.801648872Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.801+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*QgPs5o count=4310","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224801]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6J2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.801863083Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.801+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*QgPs5o count=4311","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224801]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6Z2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.801871215Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.801+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*QgPs5o peers=1 count=4312","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224801]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6p2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.801900154Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.801+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*QgPs5o inFlight=false count=4313","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224801]},{"_index":"continuous-tests-pods-2023.11.09","_id":"652EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.801976872Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.801+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*QgPs5o count=4314","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224801]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7J2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.801988362Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.801+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4315","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224801]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7Z2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.802135763Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.801+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4316","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224802]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7p2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.802157115Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.802+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*QgPs5o count=4317","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224802]},{"_index":"continuous-tests-pods-2023.11.09","_id":"752EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.802180878Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.802+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*QgPs5o inFlight=true count=4318","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224802]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8J2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.802193030Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.802+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*QgPs5o count=4319","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224802]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8Z2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.802281180Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.802+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4320","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224802]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8p2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.802381046Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.802+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4321","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224802]},{"_index":"continuous-tests-pods-2023.11.09","_id":"852EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.802422060Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.802+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4322","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224802]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9J2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.802574469Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.802+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*QgPs5o count=4323","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224802]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9Z2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.803446140Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.803+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4324","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224803]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9p2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.803528443Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.803+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*JrTkBJ inflight=0 count=4325","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224803]},{"_index":"continuous-tests-pods-2023.11.09","_id":"952EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.815915935Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.815+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4326","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224815]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-J2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.815954918Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.815+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4327","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224815]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-Z2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.816603455Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.816+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4328","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224816]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-p2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.816633051Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.816+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/5o/zb2rhXi9YQaBM69GSzmczDCEzDpQsH5V1bXnAMDa2GBQgPs5o + cid=zb2*QgPs5o count=4329","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224816]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-52EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.817048154Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.816+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=4176068 cid=zb2*QgPs5o count=4330","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224817]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_J2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.822970777Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.822+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4331","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224822]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_Z2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.823008280Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.822+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*QgPs5o count=4332","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224823]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_p2EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.823172353Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.822+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=20919 count=4333","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224823]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_52EtosBOvK4VyD1mLou","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.823195100Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.822+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4334","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224823]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.823200146Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.822+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*QgPs5o queue=0 count=4335","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224823]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.823204136Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.823+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4336","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224823]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ap2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.823207980Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.823+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4337","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224823]},{"_index":"continuous-tests-pods-2023.11.09","_id":"A52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.823232433Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.823+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4338","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224823]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.823404023Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.823+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*QgPs5o count=4339","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224823]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.823583805Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.823+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*QgPs5o inflight=1 count=4340","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224823]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Bp2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.823593987Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.823+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*QgPs5o count=4341","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224823]},{"_index":"continuous-tests-pods-2023.11.09","_id":"B52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.823951738Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.823+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=62 cid=zb2*QgPs5o bytes=65472 blockOffset=0 count=4342","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224823]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.824368239Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.823+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4343","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224824]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.824386321Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.824+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4344","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224824]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Cp2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.824392290Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.824+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*7q4u6V count=4345","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224824]},{"_index":"continuous-tests-pods-2023.11.09","_id":"C52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.824406242Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.824+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*7q4u6V count=4346","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224824]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.824415103Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.824+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*7q4u6V peers=1 count=4347","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224824]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.824534184Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.824+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*7q4u6V inFlight=false count=4348","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224824]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Dp2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.824573157Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.824+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*7q4u6V count=4349","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224824]},{"_index":"continuous-tests-pods-2023.11.09","_id":"D52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.824588092Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.824+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4350","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224824]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.824634423Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.824+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4351","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224824]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.824641360Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.824+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*7q4u6V count=4352","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224824]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ep2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.824759670Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.824+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*7q4u6V inFlight=true count=4353","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224824]},{"_index":"continuous-tests-pods-2023.11.09","_id":"E52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.824788960Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.824+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*7q4u6V count=4354","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224824]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.824812759Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.824+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4355","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224824]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.824877227Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.824+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4356","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224824]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Fp2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.824896096Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.824+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4357","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224824]},{"_index":"continuous-tests-pods-2023.11.09","_id":"F52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.825170796Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.825+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*7q4u6V count=4358","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224825]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.825491800Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.825+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4359","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224825]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.825517653Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.825+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*QgPs5o inflight=0 count=4360","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224825]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Gp2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.838147808Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.837+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4361","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224838]},{"_index":"continuous-tests-pods-2023.11.09","_id":"G52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.838213023Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.837+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4362","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224838]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.839364024Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.839+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4363","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224839]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.839400933Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.839+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/6V/zb2rhYUxdvJ143YhrcUwapptmPBnPqNa4iDig9XGgEj7q4u6V + cid=zb2*7q4u6V count=4364","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224839]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Hp2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.839971034Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.839+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=4241540 cid=zb2*7q4u6V count=4365","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224839]},{"_index":"continuous-tests-pods-2023.11.09","_id":"H52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.845919852Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.845+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4366","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224845]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.845955149Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.845+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*7q4u6V count=4367","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224845]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.845984535Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.845+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=21379 count=4368","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224845]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ip2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.846010374Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.845+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4369","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224846]},{"_index":"continuous-tests-pods-2023.11.09","_id":"I52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.846021358Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.845+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*7q4u6V queue=0 count=4370","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224846]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.846050869Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.845+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4371","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224846]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.846143588Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.846+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4372","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224846]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Jp2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.846175699Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.846+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4373","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224846]},{"_index":"continuous-tests-pods-2023.11.09","_id":"J52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.846325498Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.846+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*7q4u6V count=4374","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224846]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.846556793Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.846+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*7q4u6V inflight=1 count=4375","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224846]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.846653502Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.846+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*7q4u6V count=4376","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224846]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Kp2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.846822690Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.846+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=63 cid=zb2*7q4u6V bytes=65472 blockOffset=0 count=4377","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224846]},{"_index":"continuous-tests-pods-2023.11.09","_id":"K52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.846836320Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.846+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4378","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224846]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.847148013Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.847+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4379","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224847]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.847207198Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.847+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*yqzpgL count=4380","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224847]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Lp2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.847352048Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.847+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*yqzpgL count=4381","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224847]},{"_index":"continuous-tests-pods-2023.11.09","_id":"L52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.847378142Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.847+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*yqzpgL peers=1 count=4382","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224847]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.847496994Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.847+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*yqzpgL inFlight=false count=4383","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224847]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.847515379Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.847+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*yqzpgL count=4384","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224847]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Mp2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.847523861Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.847+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4385","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224847]},{"_index":"continuous-tests-pods-2023.11.09","_id":"M52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.847677451Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.847+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4386","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224847]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.847684363Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.847+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*yqzpgL count=4387","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224847]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.847688743Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.847+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*yqzpgL inFlight=true count=4388","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224847]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Np2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.847696543Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.847+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*yqzpgL count=4389","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224847]},{"_index":"continuous-tests-pods-2023.11.09","_id":"N52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.847729409Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.847+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4390","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224847]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.847789504Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.847+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4391","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224847]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.847860312Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.847+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4392","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224847]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Op2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.848033941Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.847+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*yqzpgL count=4393","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224848]},{"_index":"continuous-tests-pods-2023.11.09","_id":"O52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.849997737Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.849+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4394","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224849]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.850109076Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.849+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*7q4u6V inflight=0 count=4395","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224850]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.861897178Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.861+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4396","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224861]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Pp2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.861966031Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.861+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4397","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224861]},{"_index":"continuous-tests-pods-2023.11.09","_id":"P52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.862641342Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.862+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4398","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224862]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.862769347Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.862+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/gL/zb2rhmQS7ysdKY8fy4trkkbvhF4NhrJqLnaC1p13yq6yqzpgL + cid=zb2*yqzpgL count=4399","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224862]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.863150428Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.863+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=4307012 cid=zb2*yqzpgL count=4400","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224863]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Qp2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.868716042Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.868+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4401","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224868]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Q52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.868809878Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.868+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*yqzpgL count=4402","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224868]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.868830282Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.868+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=21287 count=4403","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224868]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.868835034Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.868+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4404","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224868]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Rp2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.868839784Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.868+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*yqzpgL queue=0 count=4405","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224868]},{"_index":"continuous-tests-pods-2023.11.09","_id":"R52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.868912488Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.868+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4406","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224868]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.869012142Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.868+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4407","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224869]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.869020452Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.868+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4408","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224869]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Sp2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.869293543Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.869+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*yqzpgL count=4409","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224869]},{"_index":"continuous-tests-pods-2023.11.09","_id":"S52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.869389663Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.869+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*yqzpgL inflight=1 count=4410","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224869]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.869469537Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.869+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*yqzpgL count=4411","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224869]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.869671827Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.869+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=64 cid=zb2*yqzpgL bytes=65472 blockOffset=0 count=4412","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224869]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Tp2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.869679293Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.869+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4413","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224869]},{"_index":"continuous-tests-pods-2023.11.09","_id":"T52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.869981128Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.869+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4414","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224869]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.870038866Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.869+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*MJzNcz count=4415","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224870]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.870208405Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.870+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*MJzNcz count=4416","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224870]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Up2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.870258819Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.870+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*MJzNcz peers=1 count=4417","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224870]},{"_index":"continuous-tests-pods-2023.11.09","_id":"U52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.870272392Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.870+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*MJzNcz inFlight=false count=4418","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224870]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.870401585Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.870+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*MJzNcz count=4419","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224870]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.870453433Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.870+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4420","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224870]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Vp2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.870471516Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.870+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4421","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224870]},{"_index":"continuous-tests-pods-2023.11.09","_id":"V52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.870591011Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.870+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*MJzNcz count=4422","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224870]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.870626541Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.870+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*MJzNcz inFlight=true count=4423","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224870]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.870647979Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.870+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*MJzNcz count=4424","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224870]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Wp2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.870691251Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.870+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4425","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224870]},{"_index":"continuous-tests-pods-2023.11.09","_id":"W52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.870744995Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.870+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4426","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224870]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.870931345Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.870+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4427","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224870]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.871135614Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.870+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*MJzNcz count=4428","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224871]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Xp2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.871794100Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.871+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4429","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224871]},{"_index":"continuous-tests-pods-2023.11.09","_id":"X52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.871822222Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.871+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*yqzpgL inflight=0 count=4430","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224871]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.883962198Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.883+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4431","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224883]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.884029729Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.883+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4432","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224884]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Yp2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.884525524Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.884+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4433","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224884]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Y52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.884645879Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.884+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/cz/zb2rhi32ZD1FB5HU59SY8NinKi2jTN2EHDum1vZhkjiMJzNcz + cid=zb2*MJzNcz count=4434","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224884]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.885129532Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.884+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=4372484 cid=zb2*MJzNcz count=4435","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224885]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.890714930Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.890+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4436","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224890]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Zp2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.890766364Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.890+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*MJzNcz count=4437","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224890]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Z52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.890772529Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.890+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=20361 count=4438","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224890]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.890840224Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.890+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4439","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224890]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.890861958Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.890+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*MJzNcz queue=0 count=4440","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224890]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ap2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.890978379Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.890+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4441","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224890]},{"_index":"continuous-tests-pods-2023.11.09","_id":"a52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.891013760Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.890+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4442","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224891]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.891042668Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.890+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4443","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224891]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bZ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.891310892Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.891+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*MJzNcz count=4444","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224891]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bp2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.891428674Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.891+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*MJzNcz inflight=1 count=4445","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224891]},{"_index":"continuous-tests-pods-2023.11.09","_id":"b52EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.891438944Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.891+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*MJzNcz count=4446","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224891]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cJ2EtosBOvK4VyD1mLsu","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.891712235Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.891+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=65 cid=zb2*MJzNcz bytes=65472 blockOffset=0 count=4447","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224891]},{"_index":"continuous-tests-pods-2023.11.09","_id":"752EtosBOvK4VyD1mb8i","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.891983514Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.891+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4449","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224891]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7p2EtosBOvK4VyD1mb8i","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.891728653Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.891+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4448","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224891]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8J2EtosBOvK4VyD1mb8i","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.891998686Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.891+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*LGyPLc count=4450","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224891]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8p2EtosBOvK4VyD1mb8i","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.892164383Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.892+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*LGyPLc peers=1 count=4452","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224892]},{"_index":"continuous-tests-pods-2023.11.09","_id":"852EtosBOvK4VyD1mb8i","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.892251231Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.892+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*LGyPLc inFlight=false count=4453","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224892]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9J2EtosBOvK4VyD1mb8i","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.892312463Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.892+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*LGyPLc count=4454","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224892]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9Z2EtosBOvK4VyD1mb8i","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.892336561Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.892+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4455","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224892]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-Z2EtosBOvK4VyD1mb8i","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.892491689Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.892+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*LGyPLc count=4459","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224892]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-p2EtosBOvK4VyD1mb8i","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.892530425Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.892+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4460","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224892]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-52EtosBOvK4VyD1mb8i","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.892576586Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.892+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4461","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224892]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9p2EtosBOvK4VyD1mb8i","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.892342808Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.892+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4456","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224892]},{"_index":"continuous-tests-pods-2023.11.09","_id":"952EtosBOvK4VyD1mb8i","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.892368123Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.892+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*LGyPLc count=4457","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224892]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-J2EtosBOvK4VyD1mb8i","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.892428637Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.892+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*LGyPLc inFlight=true count=4458","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224892]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8Z2EtosBOvK4VyD1mb8i","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.892154853Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.892+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*LGyPLc count=4451","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224892]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_J2EtosBOvK4VyD1mb8i","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.892594557Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.892+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4462","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224892]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_Z2EtosBOvK4VyD1mb8i","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.892812429Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.892+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*LGyPLc count=4463","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224892]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_p2EtosBOvK4VyD1mb8i","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.893396522Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.893+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4464","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224893]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_52EtosBOvK4VyD1mb8i","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.893525252Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.893+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*MJzNcz inflight=0 count=4465","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224893]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.904525845Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.904+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4466","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224904]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.904614348Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.904+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4467","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224904]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ap2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.905367514Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.905+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4468","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224905]},{"_index":"continuous-tests-pods-2023.11.09","_id":"A52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.905417646Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.905+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/Lc/zb2rhmXXQHc6nnwHNeDzt77MQw3kVBCbXovEbKHjpXTLGyPLc + cid=zb2*LGyPLc count=4469","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224905]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.905967011Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.905+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=4437956 cid=zb2*LGyPLc count=4470","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224905]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.911390189Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.911+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*LGyPLc queue=0 count=4475","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224911]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Cp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.911399633Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.911+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4476","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224911]},{"_index":"continuous-tests-pods-2023.11.09","_id":"C52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.911441541Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.911+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4477","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224911]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.911447975Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.911+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4478","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224911]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.911775297Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.911+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*LGyPLc count=4479","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224911]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.911289575Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.910+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4471","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224911]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Bp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.911320668Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.910+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*LGyPLc count=4472","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224911]},{"_index":"continuous-tests-pods-2023.11.09","_id":"B52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.911326775Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.911+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=18980 count=4473","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224911]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.911382364Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.911+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4474","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224911]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Dp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.911857982Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.911+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*LGyPLc inflight=1 count=4480","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224911]},{"_index":"continuous-tests-pods-2023.11.09","_id":"D52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.911921977Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.911+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*LGyPLc count=4481","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224911]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.912081921Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.911+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=66 cid=zb2*LGyPLc bytes=65472 blockOffset=0 count=4482","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.912094424Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.912+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4483","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ep2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.912289991Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.912+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4484","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"E52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.912351584Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.912+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*BUQCFY count=4485","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.912472088Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.912+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*BUQCFY count=4486","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.912483177Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.912+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*BUQCFY peers=1 count=4487","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Fp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.912581075Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.912+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*BUQCFY inFlight=false count=4488","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"F52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.912612414Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.912+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*BUQCFY count=4489","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.912657482Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.912+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4491","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"G52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.912787931Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.912+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*BUQCFY inFlight=true count=4493","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.912813648Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.912+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*BUQCFY count=4494","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.912860837Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.912+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4495","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.912626198Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.912+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4490","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Gp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.912741453Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.912+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*BUQCFY count=4492","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Hp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.912876013Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.912+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4496","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"H52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.912988875Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.912+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4497","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224912]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.913167671Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.913+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*BUQCFY count=4498","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224913]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.914142092Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.913+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4499","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224914]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ip2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.914165085Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.913+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*LGyPLc inflight=0 count=4500","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224914]},{"_index":"continuous-tests-pods-2023.11.09","_id":"I52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.927309796Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.926+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4501","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224927]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.927350500Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.927+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4502","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224927]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.927927469Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.927+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4503","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224927]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Jp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.928084125Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.927+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/FY/zb2rhXqeJUb1ehkK4Q7XAy3WHSdWZQcAnY3izWTL7WjBUQCFY + cid=zb2*BUQCFY count=4504","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224928]},{"_index":"continuous-tests-pods-2023.11.09","_id":"J52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.928435961Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.928+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=4503428 cid=zb2*BUQCFY count=4505","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224928]},{"_index":"continuous-tests-pods-2023.11.09","_id":"K52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.934172504Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.934+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4509","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224934]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.934178229Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.934+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*BUQCFY queue=0 count=4510","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224934]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.934253035Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.934+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4511","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224934]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Lp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.934267259Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.934+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4512","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224934]},{"_index":"continuous-tests-pods-2023.11.09","_id":"L52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.934297494Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.934+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4513","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224934]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.934025193Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.933+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4506","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224934]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.934059008Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.933+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*BUQCFY count=4507","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224934]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Kp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.934132830Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.933+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=21514 count=4508","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224934]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.934601623Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.934+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*BUQCFY count=4514","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224934]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.935690919Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.935+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*v22oiY peers=1 count=4522","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224935]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.935826600Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.935+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*v22oiY inFlight=false count=4523","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224935]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Op2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.935838901Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.935+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*v22oiY count=4524","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224935]},{"_index":"continuous-tests-pods-2023.11.09","_id":"O52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.935843909Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.935+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4525","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224935]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.935848909Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.935+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4526","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224935]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.935258855Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.934+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*BUQCFY inflight=1 count=4515","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224935]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Mp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.935313996Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.934+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*BUQCFY count=4516","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224935]},{"_index":"continuous-tests-pods-2023.11.09","_id":"M52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.935321558Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.935+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=67 cid=zb2*BUQCFY bytes=65472 blockOffset=0 count=4517","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224935]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.935326393Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.935+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4518","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224935]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.935413377Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.935+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4519","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224935]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Np2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.935551311Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.935+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*v22oiY count=4520","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224935]},{"_index":"continuous-tests-pods-2023.11.09","_id":"N52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.935623219Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.935+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*v22oiY count=4521","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224935]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.935894885Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.935+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*v22oiY count=4527","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224935]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Pp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.935922864Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.935+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*v22oiY inFlight=true count=4528","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224935]},{"_index":"continuous-tests-pods-2023.11.09","_id":"P52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.935963011Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.935+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*v22oiY count=4529","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224935]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.935987671Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.935+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4530","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224935]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.936075826Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.935+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4531","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224936]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Qp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.936219227Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.936+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4532","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224936]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Q52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.936387189Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.936+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*v22oiY count=4533","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224936]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.936748788Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.936+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4534","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224936]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.936911744Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.936+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*BUQCFY inflight=0 count=4535","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224936]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Rp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.949807648Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.949+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4536","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224949]},{"_index":"continuous-tests-pods-2023.11.09","_id":"R52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.949851053Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.949+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4537","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224949]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.950287584Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.950+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4538","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224950]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.950548691Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.950+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/iY/zb2rhhikSdyz8upB2p99htZB5bJdbTMrnTcJxgfSieFv22oiY + cid=zb2*v22oiY count=4539","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224950]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Sp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.951270802Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.951+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=4568900 cid=zb2*v22oiY count=4540","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224951]},{"_index":"continuous-tests-pods-2023.11.09","_id":"S52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.957261164Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.957+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4541","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224957]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.957320638Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.957+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*v22oiY count=4542","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224957]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Tp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.957403111Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.957+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4544","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224957]},{"_index":"continuous-tests-pods-2023.11.09","_id":"T52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.957429574Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.957+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*v22oiY queue=0 count=4545","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224957]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Up2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.957610668Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.957+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4548","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224957]},{"_index":"continuous-tests-pods-2023.11.09","_id":"U52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.957766920Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.957+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*v22oiY count=4549","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224957]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.957337884Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.957+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=21627 count=4543","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224957]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.957521537Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.957+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4546","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224957]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.957586233Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.957+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4547","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224957]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.958039186Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.957+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*v22oiY inflight=1 count=4550","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224958]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.958064355Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.957+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*v22oiY count=4551","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224958]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Vp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.958297603Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.958+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=68 cid=zb2*v22oiY bytes=65472 blockOffset=0 count=4552","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224958]},{"_index":"continuous-tests-pods-2023.11.09","_id":"V52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.958401689Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.958+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4553","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224958]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.958770445Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.958+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4554","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224958]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.958782571Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.958+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*aagaW3 count=4555","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224958]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Wp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.958901248Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.958+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*aagaW3 count=4556","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224958]},{"_index":"continuous-tests-pods-2023.11.09","_id":"W52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.959069294Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.958+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*aagaW3 peers=1 count=4557","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224959]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.959077611Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.959+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*aagaW3 inFlight=false count=4558","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224959]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.959150619Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.959+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*aagaW3 count=4559","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224959]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Xp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.959156070Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.959+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4560","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224959]},{"_index":"continuous-tests-pods-2023.11.09","_id":"X52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.959179506Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.959+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4561","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224959]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.959274454Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.959+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*aagaW3 count=4562","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224959]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.959303797Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.959+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*aagaW3 inFlight=true count=4563","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224959]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Yp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.959385777Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.959+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*aagaW3 count=4564","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224959]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Y52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.959390757Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.959+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4565","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224959]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.959488124Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.959+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4566","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224959]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.959518024Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.959+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4567","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224959]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Zp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.959719644Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.959+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*aagaW3 count=4568","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224959]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.960380184Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.960+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*v22oiY inflight=0 count=4570","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224960]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Z52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.960326449Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.960+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4569","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224960]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.972102017Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.971+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4571","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224972]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ap2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.972137030Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.971+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4572","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224972]},{"_index":"continuous-tests-pods-2023.11.09","_id":"a52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.972828593Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.972+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4573","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224972]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.972952721Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.972+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/W3/zb2rhb7xUBXnw6624xAQiPV88iS8vs4EmnXYHUuqV25aagaW3 + cid=zb2*aagaW3 count=4574","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224972]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.973551262Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.973+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=4634372 cid=zb2*aagaW3 count=4575","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224973]},{"_index":"continuous-tests-pods-2023.11.09","_id":"b52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.978840669Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.978+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*aagaW3 count=4577","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224978]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.978813098Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.978+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4576","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224978]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.978942262Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.978+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=19722 count=4578","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224978]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.978979740Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.978+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4579","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224978]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.979016101Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.978+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*aagaW3 queue=0 count=4580","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224979]},{"_index":"continuous-tests-pods-2023.11.09","_id":"c52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.979059287Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.979+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4581","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224979]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.979089491Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.979+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4582","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224979]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.979125673Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.979+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4583","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224979]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.979443524Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.979+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*aagaW3 count=4584","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224979]},{"_index":"continuous-tests-pods-2023.11.09","_id":"d52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.979733981Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.979+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*aagaW3 inflight=1 count=4585","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224979]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.979753377Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.979+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*aagaW3 count=4586","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224979]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.979836419Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.979+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=69 cid=zb2*aagaW3 bytes=65472 blockOffset=0 count=4587","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224979]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ep2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.979959863Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.979+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4588","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224979]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.980173190Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.980+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*xocu7C count=4590","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224980]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.980745552Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.980+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*xocu7C count=4599","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224980]},{"_index":"continuous-tests-pods-2023.11.09","_id":"h52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.980755937Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.980+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4601","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224980]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.980763054Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.980+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4602","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224980]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.980930325Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.980+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*xocu7C count=4603","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224980]},{"_index":"continuous-tests-pods-2023.11.09","_id":"e52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.980156347Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.980+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4589","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224980]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.980324146Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.980+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*xocu7C count=4591","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224980]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.980334261Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.980+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*xocu7C peers=1 count=4592","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224980]},{"_index":"continuous-tests-pods-2023.11.09","_id":"f52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.980392481Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.980+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*xocu7C inFlight=false count=4593","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224980]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.980425291Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.980+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*xocu7C count=4594","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224980]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.980523399Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.980+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4595","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224980]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.980532184Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.980+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4596","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224980]},{"_index":"continuous-tests-pods-2023.11.09","_id":"g52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.980540791Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.980+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*xocu7C count=4597","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224980]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.980584713Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.980+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*xocu7C inFlight=true count=4598","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224980]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.980752029Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.980+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4600","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224980]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ip2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.981412010Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.981+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4604","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224981]},{"_index":"continuous-tests-pods-2023.11.09","_id":"i52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.981425969Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.981+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*aagaW3 inflight=0 count=4605","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224981]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.995629541Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.995+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4606","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224995]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.995667411Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.995+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4607","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224995]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.996160275Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.996+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4608","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224996]},{"_index":"continuous-tests-pods-2023.11.09","_id":"j52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.996314363Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.996+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/7C/zb2rhYgHqR6okboop7vNNCvzi7Yg4We4KGfH7xTJUizxocu7C + cid=zb2*xocu7C count=4609","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224996]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:04.996926578Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:04.996+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=4699844 cid=zb2*xocu7C count=4610","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574224996]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.001945472Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.001+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*xocu7C queue=0 count=4615","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225001]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.001789297Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.001+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4611","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225001]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.001818278Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.001+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*xocu7C count=4612","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225001]},{"_index":"continuous-tests-pods-2023.11.09","_id":"k52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.001867629Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.001+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=21388 count=4613","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225001]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.001894368Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.001+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4614","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225001]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.002821554Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.002+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=70 cid=zb2*xocu7C bytes=65472 blockOffset=0 count=4622","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225002]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.002860151Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.002+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4623","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225002]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.002003747Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.001+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4616","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225002]},{"_index":"continuous-tests-pods-2023.11.09","_id":"l52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.002033608Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.001+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4617","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225002]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.002165492Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.001+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4618","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225002]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.002350324Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.002+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*xocu7C count=4619","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225002]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.002544390Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.002+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*xocu7C inflight=1 count=4620","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225002]},{"_index":"continuous-tests-pods-2023.11.09","_id":"m52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.002580925Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.002+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*xocu7C count=4621","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225002]},{"_index":"continuous-tests-pods-2023.11.09","_id":"np2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.003273424Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.003+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4624","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225003]},{"_index":"continuous-tests-pods-2023.11.09","_id":"n52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.003316204Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.003+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*3KVTQS count=4625","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225003]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.003438354Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.003+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*3KVTQS count=4626","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225003]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.003499661Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.003+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*3KVTQS peers=1 count=4627","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225003]},{"_index":"continuous-tests-pods-2023.11.09","_id":"op2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.003542775Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.003+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*3KVTQS inFlight=false count=4628","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225003]},{"_index":"continuous-tests-pods-2023.11.09","_id":"o52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.003604761Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.003+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*3KVTQS count=4629","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225003]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.003651733Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.003+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4630","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225003]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.003659980Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.003+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4631","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225003]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.003842582Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.003+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*3KVTQS count=4632","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225003]},{"_index":"continuous-tests-pods-2023.11.09","_id":"p52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.003851215Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.003+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*3KVTQS inFlight=true count=4633","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225003]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.003855053Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.003+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*3KVTQS count=4634","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225003]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.003859237Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.003+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4635","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225003]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.003898430Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.003+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4636","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225003]},{"_index":"continuous-tests-pods-2023.11.09","_id":"q52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.003934783Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.003+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4637","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225003]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.004383165Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.004+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4639","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225004]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.004475095Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.004+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*xocu7C inflight=0 count=4640","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225004]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.004093892Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.004+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*3KVTQS count=4638","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225004]},{"_index":"continuous-tests-pods-2023.11.09","_id":"r52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.020159071Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.019+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4641","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225020]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.020192027Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.020+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4642","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225020]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.020715Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.020+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4643","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225020]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.020885993Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.020+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/QS/zb2rhmE9YUGHGFVWDnK8TdC3ZF67zN2XCdNUnzz6dfE3KVTQS + cid=zb2*3KVTQS count=4644","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225020]},{"_index":"continuous-tests-pods-2023.11.09","_id":"s52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.021394629Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.021+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=4765316 cid=zb2*3KVTQS count=4645","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225021]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.027202354Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.027+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=23570 count=4648","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225027]},{"_index":"continuous-tests-pods-2023.11.09","_id":"up2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.027375079Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.027+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4652","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225027]},{"_index":"continuous-tests-pods-2023.11.09","_id":"u52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.027430864Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.027+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4653","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225027]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.027534423Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.027+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*3KVTQS count=4654","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225027]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.027836880Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.027+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*3KVTQS inflight=1 count=4655","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225027]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.027127883Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.026+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4646","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225027]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.027158578Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.026+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*3KVTQS count=4647","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225027]},{"_index":"continuous-tests-pods-2023.11.09","_id":"t52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.027208048Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.027+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4649","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225027]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.027282292Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.027+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*3KVTQS queue=0 count=4650","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225027]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.027315636Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.027+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4651","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225027]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.027853804Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.027+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*3KVTQS count=4656","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225027]},{"_index":"continuous-tests-pods-2023.11.09","_id":"v52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.028038879Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.027+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=71 cid=zb2*3KVTQS bytes=65472 blockOffset=0 count=4657","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225028]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.028127483Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.028+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4658","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225028]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.028387295Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.028+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*D57Rwv count=4660","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225028]},{"_index":"continuous-tests-pods-2023.11.09","_id":"w52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.028546258Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.028+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*D57Rwv count=4661","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225028]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.028598150Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.028+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*D57Rwv peers=1 count=4662","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225028]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.028623589Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.028+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*D57Rwv inFlight=false count=4663","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225028]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.028647542Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.028+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*D57Rwv count=4664","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225028]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.028746253Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.028+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4666","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225028]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.028782772Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.028+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*D57Rwv count=4667","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225028]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.028814257Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.028+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*D57Rwv inFlight=true count=4668","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225028]},{"_index":"continuous-tests-pods-2023.11.09","_id":"y52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.028870393Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.028+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*D57Rwv count=4669","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225028]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zJ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.028966769Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.028+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4670","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225028]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.028981477Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.028+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4671","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225028]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wZ2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.028326793Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.028+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4659","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225028]},{"_index":"continuous-tests-pods-2023.11.09","_id":"x52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.028735792Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.028+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4665","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225028]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0J2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.029926197Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.029+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4674","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225029]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0Z2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.029962982Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.029+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*3KVTQS inflight=0 count=4675","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225029]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zp2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.029026236Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.028+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4672","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225029]},{"_index":"continuous-tests-pods-2023.11.09","_id":"z52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.029201397Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.029+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*D57Rwv count=4673","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225029]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0p2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.046060755Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.045+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4676","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225046]},{"_index":"continuous-tests-pods-2023.11.09","_id":"052EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.046105735Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.045+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4677","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225046]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1J2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.047133999Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.046+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4678","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225047]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1Z2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.047459628Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.047+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/wv/zb2rhX3uH7ueDTi8ktGErCJ6xwnvTMNBXMytHy1n5cpD57Rwv + cid=zb2*D57Rwv count=4679","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225047]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1p2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.048183009Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.047+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=4830788 cid=zb2*D57Rwv count=4680","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225048]},{"_index":"continuous-tests-pods-2023.11.09","_id":"152EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.055626166Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.055+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4681","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225055]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2J2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.055708274Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.055+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*D57Rwv count=4682","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225055]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2Z2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.055715145Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.055+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=26951 count=4683","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225055]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2p2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.055997022Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.055+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4684","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225055]},{"_index":"continuous-tests-pods-2023.11.09","_id":"252EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.056018978Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.055+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*D57Rwv queue=0 count=4685","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225056]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3J2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.056034349Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.055+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4686","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225056]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3Z2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.056175733Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.056+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4687","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225056]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3p2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.056185429Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.056+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4688","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225056]},{"_index":"continuous-tests-pods-2023.11.09","_id":"352EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.056491081Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.056+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*D57Rwv count=4689","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225056]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4J2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.056593889Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.056+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*D57Rwv inflight=1 count=4690","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225056]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4Z2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.056600854Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.056+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*D57Rwv count=4691","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225056]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4p2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.056841370Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.056+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=72 cid=zb2*D57Rwv bytes=65472 blockOffset=0 count=4692","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225056]},{"_index":"continuous-tests-pods-2023.11.09","_id":"452EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.056904725Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.056+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4693","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225056]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5J2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.057057430Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.056+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4694","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5Z2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.057063187Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.057+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*eHoc59 count=4695","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5p2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.057241927Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.057+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*eHoc59 count=4696","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"552EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.057247288Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.057+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*eHoc59 peers=1 count=4697","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6J2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.057285847Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.057+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*eHoc59 inFlight=false count=4698","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6Z2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.057460357Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.057+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*eHoc59 count=4699","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6p2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.057552688Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.057+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4700","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"752EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.057777226Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.057+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4705","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"652EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.057561119Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.057+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4701","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7J2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.057573716Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.057+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*eHoc59 count=4702","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7Z2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.057585719Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.057+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*eHoc59 inFlight=true count=4703","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7p2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.057755419Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.057+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*eHoc59 count=4704","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8J2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.057835842Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.057+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4706","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8Z2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.057926917Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.057+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4707","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225057]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9J2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.058799066Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.058+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*D57Rwv inflight=0 count=4710","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225058]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8p2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.058253363Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.058+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*eHoc59 count=4708","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225058]},{"_index":"continuous-tests-pods-2023.11.09","_id":"852EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.058611146Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.058+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4709","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225058]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9Z2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.073017278Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.072+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4711","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225073]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9p2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.073056198Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.072+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4712","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225073]},{"_index":"continuous-tests-pods-2023.11.09","_id":"952EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.073865476Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.073+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4713","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225073]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-J2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.073906779Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.073+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/59/zb2rhY77iKCSbZmNwWgaDfdcrwtGKArnRiikiMq6t1ueHoc59 + cid=zb2*eHoc59 count=4714","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225073]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-Z2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.074602926Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.074+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=4896260 cid=zb2*eHoc59 count=4715","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225074]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-p2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.080733928Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.080+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4716","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225080]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.080769931Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.080+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*eHoc59 count=4717","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225080]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_J2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.080786910Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.080+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=23348 count=4718","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225080]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_Z2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.080796716Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.080+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4719","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225080]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_p2EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.080923719Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.080+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*eHoc59 queue=0 count=4720","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225080]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_52EtosBOvK4VyD1mcAi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.081067083Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.080+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4721","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225081]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AJ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.081175226Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.081+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4722","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225081]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AZ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.081367875Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.081+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4723","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225081]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ap2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.081582098Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.081+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*eHoc59 count=4724","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225081]},{"_index":"continuous-tests-pods-2023.11.09","_id":"A52EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.081713213Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.081+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*eHoc59 inflight=1 count=4725","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225081]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BJ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.081740296Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.081+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*eHoc59 count=4726","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225081]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BZ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.081983119Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.081+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=73 cid=zb2*eHoc59 bytes=65472 blockOffset=0 count=4727","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225081]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Bp2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.082010549Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.081+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4728","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225082]},{"_index":"continuous-tests-pods-2023.11.09","_id":"B52EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.082392876Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.082+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4729","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225082]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CJ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.082412520Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.082+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*ZzEmaM count=4730","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225082]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CZ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.082536411Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.082+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*ZzEmaM count=4731","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225082]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Cp2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.082542426Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.082+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*ZzEmaM peers=1 count=4732","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225082]},{"_index":"continuous-tests-pods-2023.11.09","_id":"C52EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.082675462Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.082+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*ZzEmaM inFlight=false count=4733","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225082]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DJ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.082682052Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.082+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*ZzEmaM count=4734","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225082]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DZ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.082695065Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.082+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4735","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225082]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Dp2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.082701332Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.082+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4736","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225082]},{"_index":"continuous-tests-pods-2023.11.09","_id":"D52EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.082778676Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.082+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*ZzEmaM count=4737","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225082]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EJ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.082791438Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.082+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*ZzEmaM inFlight=true count=4738","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225082]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EZ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.082861261Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.082+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*ZzEmaM count=4739","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225082]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ep2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.082867307Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.082+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4740","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225082]},{"_index":"continuous-tests-pods-2023.11.09","_id":"E52EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.082958316Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.082+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4741","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225082]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FJ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.083048739Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.082+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4742","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225083]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FZ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.083192317Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.083+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*ZzEmaM count=4743","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225083]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Fp2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.084078951Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.083+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4744","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225084]},{"_index":"continuous-tests-pods-2023.11.09","_id":"F52EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.084114929Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.084+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*eHoc59 inflight=0 count=4745","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225084]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GJ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.096332620Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.096+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4746","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225096]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GZ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.096377591Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.096+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4747","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225096]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Gp2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.097121071Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.096+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4748","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225097]},{"_index":"continuous-tests-pods-2023.11.09","_id":"G52EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.097278189Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.097+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/aM/zb2rhiJQFQ78shtur4iV1wN82c9GA5RVf3vw5QnCKNaZzEmaM + cid=zb2*ZzEmaM count=4749","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225097]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HJ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.097809281Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.097+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=4961732 cid=zb2*ZzEmaM count=4750","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225097]},{"_index":"continuous-tests-pods-2023.11.09","_id":"HZ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.103757892Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.103+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4751","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225103]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Hp2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.103792705Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.103+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*ZzEmaM count=4752","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225103]},{"_index":"continuous-tests-pods-2023.11.09","_id":"H52EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.103798925Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.103+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=21129 count=4753","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225103]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IJ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.103813499Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.103+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4754","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225103]},{"_index":"continuous-tests-pods-2023.11.09","_id":"IZ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.103880205Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.103+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*ZzEmaM queue=0 count=4755","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225103]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ip2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.103887903Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.103+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4756","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225103]},{"_index":"continuous-tests-pods-2023.11.09","_id":"I52EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.103902753Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.103+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4757","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225103]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JJ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.103975850Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.103+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4758","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225103]},{"_index":"continuous-tests-pods-2023.11.09","_id":"JZ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.104225007Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.104+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*ZzEmaM count=4759","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225104]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Jp2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.104452991Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.104+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*ZzEmaM inflight=1 count=4760","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225104]},{"_index":"continuous-tests-pods-2023.11.09","_id":"J52EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.104662383Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.104+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*ZzEmaM count=4761","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225104]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KJ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.104691871Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.104+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=74 cid=zb2*ZzEmaM bytes=65472 blockOffset=0 count=4762","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225104]},{"_index":"continuous-tests-pods-2023.11.09","_id":"KZ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.104701231Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.104+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4763","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225104]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Kp2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.104894840Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.104+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4764","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225104]},{"_index":"continuous-tests-pods-2023.11.09","_id":"K52EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.104914727Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.104+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*JdDQ6h count=4765","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225104]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LJ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.105052241Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.104+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*JdDQ6h count=4766","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225105]},{"_index":"continuous-tests-pods-2023.11.09","_id":"LZ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.105086064Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.104+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*JdDQ6h peers=1 count=4767","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225105]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Lp2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.105097890Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.105+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*JdDQ6h inFlight=false count=4768","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225105]},{"_index":"continuous-tests-pods-2023.11.09","_id":"L52EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.105180032Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.105+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*JdDQ6h count=4769","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225105]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MJ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.105406090Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.105+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4770","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225105]},{"_index":"continuous-tests-pods-2023.11.09","_id":"MZ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.105432056Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.105+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4771","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225105]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Mp2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.105436870Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.105+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*JdDQ6h count=4772","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225105]},{"_index":"continuous-tests-pods-2023.11.09","_id":"M52EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.105441551Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.105+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*JdDQ6h inFlight=true count=4773","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225105]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NJ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.105451645Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.105+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*JdDQ6h count=4774","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225105]},{"_index":"continuous-tests-pods-2023.11.09","_id":"NZ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.105620259Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.105+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4775","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225105]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Np2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.105713102Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.105+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4776","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225105]},{"_index":"continuous-tests-pods-2023.11.09","_id":"N52EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.105801931Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.105+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4777","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225105]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OJ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.105934693Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.105+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*JdDQ6h count=4778","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225105]},{"_index":"continuous-tests-pods-2023.11.09","_id":"OZ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.106305296Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.106+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4779","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225106]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Op2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.106381634Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.106+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*ZzEmaM inflight=0 count=4780","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225106]},{"_index":"continuous-tests-pods-2023.11.09","_id":"O52EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.118793173Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.118+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4781","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225118]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PJ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.118831221Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.118+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4782","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225118]},{"_index":"continuous-tests-pods-2023.11.09","_id":"PZ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.119510868Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.119+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4783","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225119]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Pp2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.119686530Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.119+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/6h/zb2rhikPX485PL7avQVLiZKySzv3QD6uDo2gYyzZfFZJdDQ6h + cid=zb2*JdDQ6h count=4784","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225119]},{"_index":"continuous-tests-pods-2023.11.09","_id":"P52EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.120266067Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.120+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=5027204 cid=zb2*JdDQ6h count=4785","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225120]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QJ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.126011461Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.125+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4786","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225126]},{"_index":"continuous-tests-pods-2023.11.09","_id":"QZ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.126048274Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.125+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*JdDQ6h count=4787","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225126]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Qp2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.126054256Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.125+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=20833 count=4788","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225126]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Q52EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.126212656Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.125+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4789","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225126]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RJ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.126222295Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.126+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*JdDQ6h queue=0 count=4790","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225126]},{"_index":"continuous-tests-pods-2023.11.09","_id":"RZ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.126231955Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.126+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4791","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225126]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Rp2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.126236922Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.126+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4792","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225126]},{"_index":"continuous-tests-pods-2023.11.09","_id":"R52EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.126241756Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.126+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4793","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225126]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SJ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.126567551Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.126+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*JdDQ6h count=4794","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225126]},{"_index":"continuous-tests-pods-2023.11.09","_id":"SZ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.126771730Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.126+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*JdDQ6h inflight=1 count=4795","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225126]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Sp2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.126786260Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.126+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*JdDQ6h count=4796","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225126]},{"_index":"continuous-tests-pods-2023.11.09","_id":"S52EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.127135596Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.126+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=75 cid=zb2*JdDQ6h bytes=65472 blockOffset=0 count=4797","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225127]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TJ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.127297353Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.127+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4798","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225127]},{"_index":"continuous-tests-pods-2023.11.09","_id":"TZ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.127611748Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.127+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4799","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225127]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Tp2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.127627305Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.127+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*fhLaeP count=4800","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225127]},{"_index":"continuous-tests-pods-2023.11.09","_id":"T52EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.128121036Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.127+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*fhLaeP count=4801","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225128]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UJ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.128140097Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.127+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*fhLaeP peers=1 count=4802","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225128]},{"_index":"continuous-tests-pods-2023.11.09","_id":"UZ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.128146011Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.127+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*fhLaeP inFlight=false count=4803","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225128]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Up2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.128151674Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.127+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*fhLaeP count=4804","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225128]},{"_index":"continuous-tests-pods-2023.11.09","_id":"U52EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.128169566Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.128+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4805","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225128]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VJ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.128332206Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.128+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4806","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225128]},{"_index":"continuous-tests-pods-2023.11.09","_id":"VZ2EtosBOvK4VyD1mcEi","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.128473882Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.128+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*fhLaeP count=4807","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225128]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Vp2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.128605260Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.128+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*fhLaeP inFlight=true count=4808","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225128]},{"_index":"continuous-tests-pods-2023.11.09","_id":"V52EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.128811444Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.128+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*fhLaeP count=4809","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225128]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WJ2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.128932433Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.128+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4810","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225128]},{"_index":"continuous-tests-pods-2023.11.09","_id":"WZ2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.129048759Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.128+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4811","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225129]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Wp2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.129248322Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.129+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4812","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225129]},{"_index":"continuous-tests-pods-2023.11.09","_id":"W52EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.129383163Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.129+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*fhLaeP count=4813","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225129]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XJ2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.129734529Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.129+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4814","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225129]},{"_index":"continuous-tests-pods-2023.11.09","_id":"XZ2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.129897620Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.129+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*JdDQ6h inflight=0 count=4815","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225129]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Xp2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.143392025Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.143+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4816","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225143]},{"_index":"continuous-tests-pods-2023.11.09","_id":"X52EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.143431599Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.143+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4817","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225143]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YJ2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.144222876Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.144+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4818","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225144]},{"_index":"continuous-tests-pods-2023.11.09","_id":"YZ2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.144644804Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.144+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/eP/zb2rhgMeaZfttUYmxdkftG71uyNHGtatXcTymm4Nu2ufhLaeP + cid=zb2*fhLaeP count=4819","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225144]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Yp2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.145422437Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.145+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=5092676 cid=zb2*fhLaeP count=4820","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225145]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Y52EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.151066054Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.150+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4821","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225151]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZJ2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.151264641Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.151+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*fhLaeP count=4822","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225151]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ZZ2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.151307979Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.151+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=23278 count=4823","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225151]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Zp2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.151538541Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.151+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4824","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225151]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Z52EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.151570755Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.151+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*fhLaeP queue=0 count=4825","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225151]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aJ2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.151672524Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.151+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4826","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225151]},{"_index":"continuous-tests-pods-2023.11.09","_id":"aZ2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.151679688Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.151+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4827","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225151]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ap2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.151762091Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.151+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4828","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225151]},{"_index":"continuous-tests-pods-2023.11.09","_id":"a52EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.151977729Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.151+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*fhLaeP count=4829","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225151]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bJ2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.152138701Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.152+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*fhLaeP inflight=1 count=4830","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225152]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bZ2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.152153042Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.152+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*fhLaeP count=4831","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225152]},{"_index":"continuous-tests-pods-2023.11.09","_id":"bp2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.152435132Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.152+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=76 cid=zb2*fhLaeP bytes=65472 blockOffset=0 count=4832","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225152]},{"_index":"continuous-tests-pods-2023.11.09","_id":"b52EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.152453201Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.152+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4833","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225152]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cJ2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.152656692Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.152+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4834","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225152]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cZ2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.152666191Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.152+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*X1Uskv count=4835","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225152]},{"_index":"continuous-tests-pods-2023.11.09","_id":"cp2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.152911388Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.152+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*X1Uskv count=4836","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225152]},{"_index":"continuous-tests-pods-2023.11.09","_id":"c52EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.152919642Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.152+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*X1Uskv peers=1 count=4837","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225152]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dJ2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.153019169Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.152+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*X1Uskv inFlight=false count=4838","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225153]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dZ2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.153239763Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.152+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*X1Uskv count=4839","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225153]},{"_index":"continuous-tests-pods-2023.11.09","_id":"dp2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.153247851Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.153+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4840","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225153]},{"_index":"continuous-tests-pods-2023.11.09","_id":"d52EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.153252618Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.153+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4841","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225153]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eJ2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.153256340Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.153+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*X1Uskv count=4842","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225153]},{"_index":"continuous-tests-pods-2023.11.09","_id":"eZ2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.153260302Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.153+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*X1Uskv inFlight=true count=4843","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225153]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ep2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.153357535Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.153+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*X1Uskv count=4844","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225153]},{"_index":"continuous-tests-pods-2023.11.09","_id":"e52EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.153447221Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.153+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4845","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225153]},{"_index":"continuous-tests-pods-2023.11.09","_id":"fJ2EtosBOvK4VyD1mcEj","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.153464844Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.153+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4846","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225153]},{"_index":"continuous-tests-pods-2023.11.09","_id":"gp2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.153622460Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.153+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4847","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225153]},{"_index":"continuous-tests-pods-2023.11.09","_id":"g52EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.153764353Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.153+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*X1Uskv count=4848","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225153]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hJ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.154278069Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.154+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4849","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225154]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hZ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.154562614Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.154+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*fhLaeP inflight=0 count=4850","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225154]},{"_index":"continuous-tests-pods-2023.11.09","_id":"hp2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.161513223Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.161+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4851","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225161]},{"_index":"continuous-tests-pods-2023.11.09","_id":"h52EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.161630569Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.161+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4852","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225161]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iJ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.162296757Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.162+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4853","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225162]},{"_index":"continuous-tests-pods-2023.11.09","_id":"iZ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.162595310Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.162+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/kv/zb2rhjNfgHZnCtdPx5zx9HSCoHgtN1wx98NhYLbYyDtX1Uskv + cid=zb2*X1Uskv count=4854","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225162]},{"_index":"continuous-tests-pods-2023.11.09","_id":"ip2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.163165627Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.162+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=5158148 cid=zb2*X1Uskv count=4855","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225163]},{"_index":"continuous-tests-pods-2023.11.09","_id":"i52EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.173699704Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.173+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4856","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225173]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jJ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.173819850Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.173+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*X1Uskv count=4857","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225173]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jZ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.173843561Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.173+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=20778 count=4858","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225173]},{"_index":"continuous-tests-pods-2023.11.09","_id":"jp2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.173848017Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.173+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4859","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225173]},{"_index":"continuous-tests-pods-2023.11.09","_id":"j52EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.173921256Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.173+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*X1Uskv queue=0 count=4860","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225173]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kJ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.173926349Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.173+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4861","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225173]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kZ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.174057715Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.173+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4862","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225174]},{"_index":"continuous-tests-pods-2023.11.09","_id":"kp2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.174134924Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.174+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4863","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225174]},{"_index":"continuous-tests-pods-2023.11.09","_id":"k52EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.174480510Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.174+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*X1Uskv count=4864","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225174]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lJ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.174802231Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.174+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*X1Uskv inflight=1 count=4865","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225174]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lZ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.175066851Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.174+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*X1Uskv count=4866","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225175]},{"_index":"continuous-tests-pods-2023.11.09","_id":"lp2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.175269422Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.175+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=77 cid=zb2*X1Uskv bytes=65472 blockOffset=0 count=4867","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225175]},{"_index":"continuous-tests-pods-2023.11.09","_id":"l52EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.175374920Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.175+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4868","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225175]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mJ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.175657908Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.175+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4869","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225175]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mZ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.175794247Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.175+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*FALvqT count=4870","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225175]},{"_index":"continuous-tests-pods-2023.11.09","_id":"mp2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.175974548Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.175+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*FALvqT count=4871","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225175]},{"_index":"continuous-tests-pods-2023.11.09","_id":"m52EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.176186314Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.175+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*FALvqT peers=1 count=4872","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225176]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nJ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.176196744Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.176+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*FALvqT inFlight=false count=4873","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225176]},{"_index":"continuous-tests-pods-2023.11.09","_id":"nZ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.176262467Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.176+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*FALvqT count=4874","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225176]},{"_index":"continuous-tests-pods-2023.11.09","_id":"np2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.176339732Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.176+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4875","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225176]},{"_index":"continuous-tests-pods-2023.11.09","_id":"n52EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.176399160Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.176+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4876","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225176]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oJ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.176570341Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.176+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*FALvqT count=4877","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225176]},{"_index":"continuous-tests-pods-2023.11.09","_id":"oZ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.176587285Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.176+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*FALvqT inFlight=true count=4878","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225176]},{"_index":"continuous-tests-pods-2023.11.09","_id":"op2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.176672733Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.176+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*FALvqT count=4879","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225176]},{"_index":"continuous-tests-pods-2023.11.09","_id":"o52EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.176711698Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.176+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4880","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225176]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pJ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.176717577Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.176+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4881","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225176]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pZ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.176867716Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.176+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4882","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225176]},{"_index":"continuous-tests-pods-2023.11.09","_id":"pp2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.176977033Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.176+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*FALvqT count=4883","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225176]},{"_index":"continuous-tests-pods-2023.11.09","_id":"p52EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.177298413Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.177+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4884","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225177]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qJ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.177524240Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.177+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*X1Uskv inflight=0 count=4885","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225177]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qZ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.190247864Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.189+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4886","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225190]},{"_index":"continuous-tests-pods-2023.11.09","_id":"qp2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.190287847Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.190+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4887","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225190]},{"_index":"continuous-tests-pods-2023.11.09","_id":"q52EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.191489737Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.191+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4888","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225191]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rJ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.191735094Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.191+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/qT/zb2rhgRnNoZxoYDb4bQka7fVWxq8Rasb3S8X9YiVuAxFALvqT + cid=zb2*FALvqT count=4889","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225191]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rZ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.192163012Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.191+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=5223620 cid=zb2*FALvqT count=4890","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225192]},{"_index":"continuous-tests-pods-2023.11.09","_id":"rp2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.197894457Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.197+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4891","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225197]},{"_index":"continuous-tests-pods-2023.11.09","_id":"r52EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.197938335Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.197+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*FALvqT count=4892","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225197]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sJ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.197978606Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.197+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=21817 count=4893","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225197]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sZ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.198007567Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.197+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4894","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225198]},{"_index":"continuous-tests-pods-2023.11.09","_id":"sp2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.198149057Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.197+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*FALvqT queue=0 count=4895","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225198]},{"_index":"continuous-tests-pods-2023.11.09","_id":"s52EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.198277450Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.198+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4896","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225198]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tJ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.198451037Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.198+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4897","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225198]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tZ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.198460580Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.198+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4898","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225198]},{"_index":"continuous-tests-pods-2023.11.09","_id":"tp2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.198593331Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.198+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*FALvqT count=4899","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225198]},{"_index":"continuous-tests-pods-2023.11.09","_id":"t52EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.198856113Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.198+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*FALvqT inflight=1 count=4900","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225198]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uJ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.198889272Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.198+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*FALvqT count=4901","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225198]},{"_index":"continuous-tests-pods-2023.11.09","_id":"uZ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.199080385Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.198+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=78 cid=zb2*FALvqT bytes=65472 blockOffset=0 count=4902","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225199]},{"_index":"continuous-tests-pods-2023.11.09","_id":"up2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.199153291Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.199+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4903","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225199]},{"_index":"continuous-tests-pods-2023.11.09","_id":"u52EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.199399159Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.199+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4904","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225199]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vJ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.199455094Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.199+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*eN63Lc count=4905","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225199]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vZ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.199547664Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.199+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*eN63Lc count=4906","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225199]},{"_index":"continuous-tests-pods-2023.11.09","_id":"vp2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.199622258Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.199+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*eN63Lc peers=1 count=4907","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225199]},{"_index":"continuous-tests-pods-2023.11.09","_id":"v52EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.199632304Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.199+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*eN63Lc inFlight=false count=4908","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225199]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wJ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.199713957Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.199+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*eN63Lc count=4909","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225199]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wZ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.199721520Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.199+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4910","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225199]},{"_index":"continuous-tests-pods-2023.11.09","_id":"wp2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.199761550Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.199+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4911","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225199]},{"_index":"continuous-tests-pods-2023.11.09","_id":"w52EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.199846837Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.199+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*eN63Lc count=4912","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225199]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xJ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.199853524Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.199+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*eN63Lc inFlight=true count=4913","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225199]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xZ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.200014976Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.199+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*eN63Lc count=4914","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225200]},{"_index":"continuous-tests-pods-2023.11.09","_id":"xp2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.200029724Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.199+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4915","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225200]},{"_index":"continuous-tests-pods-2023.11.09","_id":"x52EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.200036614Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.199+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4916","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225200]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yJ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.200041023Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.199+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4917","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225200]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yZ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.200203416Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.200+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*eN63Lc count=4918","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225200]},{"_index":"continuous-tests-pods-2023.11.09","_id":"yp2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.200769515Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.200+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4919","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225200]},{"_index":"continuous-tests-pods-2023.11.09","_id":"y52EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.200952080Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.200+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*FALvqT inflight=0 count=4920","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225200]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zJ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.213433047Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.213+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4921","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225213]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zZ2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.213476767Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.213+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4922","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225213]},{"_index":"continuous-tests-pods-2023.11.09","_id":"zp2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.214186944Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.213+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4923","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225214]},{"_index":"continuous-tests-pods-2023.11.09","_id":"z52EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.214317498Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.214+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/Lc/zb2rhke4Hwyhz3xUyFKD4euAc8mUrhm7Br9ZnSrtuxgeN63Lc + cid=zb2*eN63Lc count=4924","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225214]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0J2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.214772722Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.214+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=5289092 cid=zb2*eN63Lc count=4925","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225214]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0Z2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.220623233Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.220+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4926","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225220]},{"_index":"continuous-tests-pods-2023.11.09","_id":"0p2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.220694337Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.220+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*eN63Lc count=4927","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225220]},{"_index":"continuous-tests-pods-2023.11.09","_id":"052EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.220700111Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.220+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=20934 count=4928","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225220]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1J2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.220720466Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.220+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4929","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225220]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1Z2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.220748074Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.220+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*eN63Lc queue=0 count=4930","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225220]},{"_index":"continuous-tests-pods-2023.11.09","_id":"1p2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.220818151Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.220+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4931","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225220]},{"_index":"continuous-tests-pods-2023.11.09","_id":"152EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.220915663Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.220+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4932","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225220]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2J2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.220971241Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.220+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4933","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225220]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2Z2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.221112330Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.221+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*eN63Lc count=4934","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225221]},{"_index":"continuous-tests-pods-2023.11.09","_id":"2p2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.221370136Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.221+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*eN63Lc inflight=1 count=4935","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225221]},{"_index":"continuous-tests-pods-2023.11.09","_id":"252EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.221376047Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.221+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*eN63Lc count=4936","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225221]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3J2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.221558574Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.221+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=79 cid=zb2*eN63Lc bytes=65472 blockOffset=0 count=4937","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225221]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3Z2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.221592502Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.221+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4938","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225221]},{"_index":"continuous-tests-pods-2023.11.09","_id":"3p2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.221923102Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.221+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4939","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225221]},{"_index":"continuous-tests-pods-2023.11.09","_id":"352EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.221935325Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.221+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*yaTgqz count=4940","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225221]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4J2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.222100037Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.222+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*yaTgqz count=4941","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225222]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4Z2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.222219940Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.222+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*yaTgqz peers=1 count=4942","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225222]},{"_index":"continuous-tests-pods-2023.11.09","_id":"4p2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.222453953Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.222+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*yaTgqz inFlight=false count=4943","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225222]},{"_index":"continuous-tests-pods-2023.11.09","_id":"452EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.222463841Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.222+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*yaTgqz count=4944","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225222]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5J2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.222467920Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.222+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4945","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225222]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5Z2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.222471949Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.222+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4946","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225222]},{"_index":"continuous-tests-pods-2023.11.09","_id":"5p2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.222521128Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.222+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*yaTgqz count=4947","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225222]},{"_index":"continuous-tests-pods-2023.11.09","_id":"552EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.222556148Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.222+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*yaTgqz inFlight=true count=4948","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225222]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6J2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.222602393Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.222+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*yaTgqz count=4949","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225222]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6Z2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.222670990Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.222+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4950","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225222]},{"_index":"continuous-tests-pods-2023.11.09","_id":"6p2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.222854487Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.222+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4951","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225222]},{"_index":"continuous-tests-pods-2023.11.09","_id":"652EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.223033500Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.222+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4952","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225223]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7J2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.223158657Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.223+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*yaTgqz count=4953","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225223]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7Z2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.223406966Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.223+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4954","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225223]},{"_index":"continuous-tests-pods-2023.11.09","_id":"7p2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.223616832Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.223+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*eN63Lc inflight=0 count=4955","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225223]},{"_index":"continuous-tests-pods-2023.11.09","_id":"752EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.237406241Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.235+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4956","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225237]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8J2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.237439453Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.235+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4957","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225237]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8Z2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.237444334Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.236+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4958","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225237]},{"_index":"continuous-tests-pods-2023.11.09","_id":"8p2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.237449632Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.236+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/qz/zb2rhhe4pNTHmh7QB6djBTRHfkQhR3GK3Qatqth55QXyaTgqz + cid=zb2*yaTgqz count=4959","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225237]},{"_index":"continuous-tests-pods-2023.11.09","_id":"852EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.237453890Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.236+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=5354564 cid=zb2*yaTgqz count=4960","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225237]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9J2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.242612131Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.242+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4961","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225242]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9Z2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.242657142Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.242+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*yaTgqz count=4962","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225242]},{"_index":"continuous-tests-pods-2023.11.09","_id":"9p2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.242820488Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.242+00:00 Block retrieval time topics=\"codex + pendingblocks\" tid=1 retrievalDurationUs=20477 count=4963","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225242]},{"_index":"continuous-tests-pods-2023.11.09","_id":"952EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.242973909Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.242+00:00 Schedule a task for new blocks topics=\"codex + blockexcengine\" tid=1 items=1 count=4964","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225242]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-J2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.243104989Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.242+00:00 Queueing provide block topics=\"codex + discoveryengine\" tid=1 cid=zb2*yaTgqz queue=0 count=4965","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225243]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-Z2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.243392997Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.243+00:00 Retrieving peer from peer context store topics=\"codex + peerctxstore\" tid=1 peer=16U*pYW64F count=4966","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225243]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-p2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.243414720Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.243+00:00 Paying for blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4967","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225243]},{"_index":"continuous-tests-pods-2023.11.09","_id":"-52EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.243510759Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.243+00:00 Blocks price topics=\"codex + peercontext\" tid=1 price=0 count=4968","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225243]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_J2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.243745038Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.243+00:00 Providing block topics=\"codex + discovery\" tid=1 cid=zb2*yaTgqz count=4969","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225243]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_Z2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.244115838Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.243+00:00 Advertising block topics=\"codex + discoveryengine\" tid=1 cid=zb2*yaTgqz inflight=1 count=4970","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225244]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_p2EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.244247813Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.244+00:00 Block handle success topics=\"codex + blockexcengine\" tid=1 cid=zb2*yaTgqz count=4971","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225244]},{"_index":"continuous-tests-pods-2023.11.09","_id":"_52EtosBOvK4VyD1msE3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.244568644Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.244+00:00 Reading bytes from store stream topics=\"codex + storestream\" tid=1 blockNum=80 cid=zb2*yaTgqz bytes=65472 blockOffset=0 count=4972","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225244]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AJ2EtosBOvK4VyD1msI3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.244644251Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.244+00:00 Sending chunk topics=\"codex + restapi\" tid=1 size=65472 count=4973","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225244]},{"_index":"continuous-tests-pods-2023.11.09","_id":"AZ2EtosBOvK4VyD1msI3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.245048756Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.244+00:00 Reading from manifest topics=\"codex + storestream\" tid=1 cid=zb2*CHxbDW blocks=1282 count=4974","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225245]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ap2EtosBOvK4VyD1msI3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.245107248Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.245+00:00 Getting block from local store or network topics=\"codex + networkstore\" tid=1 cid=zb2*f4YB7i count=4975","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225245]},{"_index":"continuous-tests-pods-2023.11.09","_id":"A52EtosBOvK4VyD1msI3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.245353136Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.245+00:00 Block not in local store topics=\"codex + networkstore\" tid=1 cid=zb2*f4YB7i count=4976","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225245]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BJ2EtosBOvK4VyD1msI3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.245508641Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.245+00:00 Begin block request topics=\"codex + blockexcengine\" tid=1 cid=zb2*f4YB7i peers=1 count=4977","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225245]},{"_index":"continuous-tests-pods-2023.11.09","_id":"BZ2EtosBOvK4VyD1msI3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.245583165Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.245+00:00 Adding pending future for block topics=\"codex + pendingblocks\" tid=1 cid=zb2*f4YB7i inFlight=false count=4978","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225245]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Bp2EtosBOvK4VyD1msI3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.245667311Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.245+00:00 Selecting peers who have topics=\"codex + blockexcengine\" tid=1 cid=zb2*f4YB7i count=4979","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225245]},{"_index":"continuous-tests-pods-2023.11.09","_id":"B52EtosBOvK4VyD1msI3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.245730499Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.245+00:00 Selected cheapest peers topics=\"codex + peerctxstore\" tid=1 peers=0 count=4980","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225245]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CJ2EtosBOvK4VyD1msI3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.245789421Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.245+00:00 No cheapest peers, selecting first in list topics=\"codex + blockexcengine\" tid=1 count=4981","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225245]},{"_index":"continuous-tests-pods-2023.11.09","_id":"CZ2EtosBOvK4VyD1msI3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.245966096Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.245+00:00 Monitoring block handle topics=\"codex + blockexcengine\" tid=1 cid=zb2*f4YB7i count=4982","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225245]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Cp2EtosBOvK4VyD1msI3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.245981264Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.245+00:00 Setting inflight topics=\"codex + pendingblocks\" tid=1 cid=zb2*f4YB7i inFlight=true count=4983","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225245]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Dp2EtosBOvK4VyD1msI3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.246500134Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.246+00:00 Sending message to remote topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4987","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225246]},{"_index":"continuous-tests-pods-2023.11.09","_id":"D52EtosBOvK4VyD1msI3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.246683960Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.246+00:00 Sending wantHave request to peers topics=\"codex + blockexcengine\" tid=1 cid=zb2*f4YB7i count=4988","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225246]},{"_index":"continuous-tests-pods-2023.11.09","_id":"C52EtosBOvK4VyD1msI3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.246075094Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.245+00:00 Sending wantBlock request to topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F cid=zb2*f4YB7i count=4984","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225246]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DJ2EtosBOvK4VyD1msI3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.246193815Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.246+00:00 Sending want list to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F type=WantBlock items=1 count=4985","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225246]},{"_index":"continuous-tests-pods-2023.11.09","_id":"DZ2EtosBOvK4VyD1msI3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.246449359Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.246+00:00 Sending message to peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F count=4986","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225246]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EJ2EtosBOvK4VyD1msI3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.247052450Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.246+00:00 Provided to nodes topics=\"codex + discovery\" tid=1 nodes=1 count=4989","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225247]},{"_index":"continuous-tests-pods-2023.11.09","_id":"EZ2EtosBOvK4VyD1msI3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.247305062Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.247+00:00 Advertised block topics=\"codex + discoveryengine\" tid=1 cid=zb2*yaTgqz inflight=0 count=4990","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225247]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Ep2EtosBOvK4VyD1msI3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.259856950Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.259+00:00 Got message for peer topics=\"codex + blockexcnetworkpeer\" tid=1 peer=16U*pYW64F count=4991","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225259]},{"_index":"continuous-tests-pods-2023.11.09","_id":"E52EtosBOvK4VyD1msI3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.259907972Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.259+00:00 Handling blocks for peer topics=\"codex + blockexcnetwork\" tid=1 peer=16U*pYW64F items=1 count=4992","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225259]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FJ2EtosBOvK4VyD1msI3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.260731892Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.260+00:00 Got blocks from peer topics=\"codex + blockexcengine\" tid=1 peer=16U*pYW64F len=1 count=4993","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225260]},{"_index":"continuous-tests-pods-2023.11.09","_id":"FZ2EtosBOvK4VyD1msI3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.260962993Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.260+00:00 Storing block with key topics=\"codex + repostore\" tid=1 key=/repo/blocks/7i/zb2rhabnSwEWB9tcCxiN675S9BwWTrLEGnEHJ1iDbMmf4YB7i + cid=zb2*f4YB7i count=4994","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225260]},{"_index":"continuous-tests-pods-2023.11.09","_id":"Fp2EtosBOvK4VyD1msI3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.261379107Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.261+00:00 Updating quota topics=\"codex + repostore\" tid=1 used=5420036 cid=zb2*f4YB7i count=4995","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225261]},{"_index":"continuous-tests-pods-2023.11.09","_id":"F52EtosBOvK4VyD1msI3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.267019753Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.266+00:00 Resolving blocks topics=\"codex + blockexcengine\" tid=1 blocks=1 count=4996","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225267]},{"_index":"continuous-tests-pods-2023.11.09","_id":"GJ2EtosBOvK4VyD1msI3","_score":null,"_source":{"@timestamp":"2023-11-09T23:57:05.267052441Z","container_image":"docker.io/codexstorage/nim-codex:sha-8c8f712-dist-tests","container_name":"codex1-3","file":"/var/log/pods/codex-continuous-tests-profiling-two-client-tests-sched-0_codex1-3-b558568cf-tvcsc_6fd429fe-ced6-487f-968a-acb1b2b4244b/codex1-3/0.log","message":"TRC + 2023-11-09 23:57:05.266+00:00 Resolving block topics=\"codex + pendingblocks\" tid=1 cid=zb2*f4YB7i count=4997","pod_ip":"10.244.3.62","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests","codexid":"docker.io-codexstorage/nim-codex:sha-8c8f712-dist-tests","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"b558568cf","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20231109-101554","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-b558568cf-tvcsc","pod_namespace":"codex-continuous-tests-profiling-two-client-tests-sched-0","pod_node_name":"pool-8vcpu-16gb-xcb2c"},"sort":[1699574225267]}]}}' + headers: + Transfer-Encoding: + - chunked + X-elastic-product: + - Elasticsearch + content-type: + - application/vnd.elasticsearch+json;compatible-with=8 + status: + code: 200 + message: OK +- request: + body: '{"scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDnF1ZXJ5VGhlbkZldGNoDhYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABna8WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABnbAWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABnbIWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABnbEWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABnbMWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABnbQWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABnbUWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABnbcWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABnbYWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABnbgWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABnbkWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABnboWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABnbsWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxYwcURrdWJxbFFWSzc4VXJMdzJCdFV3AAAAAAABnbwWVmdpQjdNT1RUM3VuSmxXekR6ZnREZw=="}' + headers: + accept: + - application/vnd.elasticsearch+json; compatible-with=8 + connection: + - keep-alive + content-type: + - application/vnd.elasticsearch+json; compatible-with=8 + user-agent: + - elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0) + x-elastic-client-meta: + - es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7 + method: DELETE + uri: http://localhost:9200/_search/scroll + response: + body: + string: '{"succeeded":true,"num_freed":14}' + headers: + X-elastic-product: + - Elasticsearch + content-length: + - '33' + content-type: + - application/vnd.elasticsearch+json;compatible-with=8 + status: + code: 200 + message: OK +version: 1 diff --git a/logtools/log/sources/input/elastic_search/tests/test_elasticsearch_source.py b/logtools/log/sources/input/elastic_search/tests/test_elasticsearch_source.py index fe822d0..9bcfd72 100644 --- a/logtools/log/sources/input/elastic_search/tests/test_elasticsearch_source.py +++ b/logtools/log/sources/input/elastic_search/tests/test_elasticsearch_source.py @@ -27,3 +27,15 @@ def test_should_fetch_logs_by_date(): assert {line.location.pod_name for line in lines} == {'codex1-3-b558568cf-tvcsc', 'bootstrap-2-58b69484bc-88msf'} assert {line.location.run_id for line in lines} == {'20231109-101554'} + +@pytest.mark.vcr +def test_should_fetch_logs_when_no_dates_are_specified(): + log = ElasticSearchSource( + run_id='20231109-101554', + pods={'codex1-3-b558568cf-tvcsc'} + ) + + try: + next(log.__iter__()) + except StopIteration: + assert False, "Should have returned at least one log line" diff --git a/logtools/log/tests/__init__.py b/logtools/log/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/logtools/log/tests/test_tree.py b/logtools/log/tests/test_tree.py new file mode 100644 index 0000000..f866339 --- /dev/null +++ b/logtools/log/tests/test_tree.py @@ -0,0 +1,16 @@ +from logtools.log.utils import tree + + +def test_should_allow_nodes_to_be_added(): + a_tree = tree() + a_tree['a']['b']['c'] = 'd' + a_tree['a']['b']['d'] = 'e' + + assert a_tree == { + 'a': { + 'b': { + 'c': 'd', + 'd': 'e' + } + } + } diff --git a/logtools/log/utils.py b/logtools/log/utils.py new file mode 100644 index 0000000..94dd0ba --- /dev/null +++ b/logtools/log/utils.py @@ -0,0 +1,9 @@ +from collections import defaultdict + +from typing import Dict + +Tree = Dict[str, 'Tree'] + + +def tree() -> Tree: + return defaultdict(tree) diff --git a/pyproject.toml b/pyproject.toml index d08fc6d..6f75648 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "logtools" -version = "1.0.0" +version = "1.0.1" description = "" authors = ["gmega "] readme = "README.md"