From 8cc04e450edef48b2326f421abf5f1d0203d27e3 Mon Sep 17 00:00:00 2001 From: gmega Date: Fri, 9 Feb 2024 15:25:59 -0300 Subject: [PATCH] add log fetch limit --- logtools/cli/es_logs.py | 19 +-- .../sources/input/elastic_search_source.py | 19 ++- ...its_when_limit_larger_than_batch_size.yaml | 132 ++++++++++++++++++ ...ts_when_limit_smaller_than_batch_size.yaml | 96 +++++++++++++ .../input/tests/test_elasticsearch_source.py | 28 ++++ pyproject.toml | 2 +- 6 files changed, 284 insertions(+), 12 deletions(-) create mode 100644 logtools/log/sources/input/tests/cassettes/test_elasticsearch_source/test_should_respect_fetching_limits_when_limit_larger_than_batch_size.yaml create mode 100644 logtools/log/sources/input/tests/cassettes/test_elasticsearch_source/test_should_respect_fetching_limits_when_limit_smaller_than_batch_size.yaml diff --git a/logtools/cli/es_logs.py b/logtools/cli/es_logs.py index 1e956f2..d56c46e 100644 --- a/logtools/cli/es_logs.py +++ b/logtools/cli/es_logs.py @@ -14,10 +14,10 @@ from rich.console import Console from rich.json import JSON from rich.table import Table +from logtools import version_string from logtools.cli.palettes import ColorMap from logtools.log.sources.input.elastic_search_source import ElasticSearchSource from logtools.resource.elastic_search_log_repo import ElasticSearchLogRepo -from logtools import version_string class ResourceType(Enum): @@ -79,21 +79,23 @@ def get_logs(args, client: Elasticsearch): ) elif resource == ResourceType.runs: run = ElasticSearchLogRepo(client=client).test_run(test_run_id=args.test_run_id).test_run - get_pod_logs(set(run.pods), client, start_date=run.start, end_date=run.end) + get_pod_logs(set(run.pods), client, limit=args.limit, start_date=run.start, end_date=run.end) def get_pod_logs(pods: Set[str], client: Elasticsearch, colored_output: bool = True, + limit: Optional[int] = None, start_date: Optional[datetime] = None, end_date: Optional[datetime] = None): colors = ColorMap() - for line in ElasticSearchSource( + for i, line in enumerate(ElasticSearchSource( pods=pods, client=client, start_date=start_date, end_date=end_date, - ): + limit=limit, + )): output = f'[{line.location.pod_name}]: {line.raw}' if colored_output: output = f'{colors[line.location.pod_name]}{output}{Style.reset}' @@ -176,6 +178,7 @@ def _add_describe_cli(subparsers): def _add_logs_cli(subparsers): logs = subparsers.add_parser('logs', help='fetch pod logs') logs.set_defaults(main=get_logs) + logs.add_argument('--limit', type=int, help='limit the number of log entries to fetch') log_subparsers = logs.add_subparsers(title='resource type', dest='resource_type', required=True) @@ -184,11 +187,11 @@ def _add_logs_cli(subparsers): pod_logs = log_subparsers.add_parser('pods', help='fetch logs for a pod') pod_logs.add_argument('pods', nargs='+', help='pod names to fetch logs from') pod_logs.add_argument('--from', dest='from_', type=tsparser.parse, - help='show entries from date/time (MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm), ' - 'treated as UTC if no timezone given', default=None) + help='show entries from date/time (MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm), ' + 'treated as UTC if no timezone given', default=None) pod_logs.add_argument('--to', dest='to', type=tsparser.parse, - help='show entries until date/time (MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm), ' - 'treated as UTC if no timezone given', default=None) + help='show entries until date/time (MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm), ' + 'treated as UTC if no timezone given', default=None) run_logs = log_subparsers.add_parser('runs', help='fetch logs for a test run') run_logs.add_argument('test_run_id', help='run ID to fetch logs from') diff --git a/logtools/log/sources/input/elastic_search_source.py b/logtools/log/sources/input/elastic_search_source.py index c290b63..36ae52a 100644 --- a/logtools/log/sources/input/elastic_search_source.py +++ b/logtools/log/sources/input/elastic_search_source.py @@ -11,6 +11,7 @@ from logtools.log.utils import tree logger = logging.getLogger(__name__) INDEX_PREFIX = 'continuous-tests-pods' +ES_MAX_BATCH_SIZE = 10_000 @dataclass @@ -29,6 +30,8 @@ class ElasticSearchSource(LogSource[TimestampedLogLine[ElasticSearchLocation]]): client: Optional[Elasticsearch] = None, start_date: Optional[datetime] = None, end_date: Optional[datetime] = None, + limit: Optional[int] = None, + es_batch_size=ES_MAX_BATCH_SIZE ): if client is None: logger.warning('No client provided, defaulting to localhost') @@ -40,6 +43,9 @@ class ElasticSearchSource(LogSource[TimestampedLogLine[ElasticSearchLocation]]): self.client = client self.start_date = start_date self.end_date = end_date + self.limit = limit + self.es_batch_size = es_batch_size + self.page_fetch_counter = 0 def __iter__(self) -> Iterator[TimestampedLogLine[ElasticSearchLocation]]: for index in self._indices(): @@ -88,8 +94,11 @@ class ElasticSearchSource(LogSource[TimestampedLogLine[ElasticSearchLocation]]): return query def _run_scan(self, query: Dict[str, Any], index: str): - # the search type stub does not contain the body argument for some reason so we disable typing here. - initial = self.client.search(index=index, body=query, size=10_000, scroll='2m') # type: ignore + remaining = self.limit if self.limit is not None else float('inf') + # XXX the search type stub does not contain the body argument for some reason so we disable typing here. + initial = self.client.search(index=index, body=query, size=min(remaining, self.es_batch_size), + scroll='2m') # type: ignore + self.page_fetch_counter += 1 scroll_id = initial['_scroll_id'] results = initial @@ -97,12 +106,16 @@ class ElasticSearchSource(LogSource[TimestampedLogLine[ElasticSearchLocation]]): while True: documents = results['hits']['hits'] if not documents: - break + return for doc in documents: yield doc + remaining -= 1 + if remaining <= 0: + return results = self.client.scroll(scroll_id=scroll_id, scroll='2m') + self.page_fetch_counter += 1 finally: self.client.clear_scroll(scroll_id=scroll_id) diff --git a/logtools/log/sources/input/tests/cassettes/test_elasticsearch_source/test_should_respect_fetching_limits_when_limit_larger_than_batch_size.yaml b/logtools/log/sources/input/tests/cassettes/test_elasticsearch_source/test_should_respect_fetching_limits_when_limit_larger_than_batch_size.yaml new file mode 100644 index 0000000..f4e6081 --- /dev/null +++ b/logtools/log/sources/input/tests/cassettes/test_elasticsearch_source/test_should_respect_fetching_limits_when_limit_larger_than_batch_size.yaml @@ -0,0 +1,132 @@ +interactions: +- request: + body: '{"query":{"bool":{"filter":[{"terms":{"pod_name.keyword":["codex1-3-6c565dbd66-wxg49"]}},{"term":{"pod_labels.runid.keyword":"20240208-115030"}}]}},"size":7,"sort":[{"@timestamp":"asc"}]}' + headers: + accept: + - application/vnd.elasticsearch+json; compatible-with=8 + connection: + - keep-alive + content-type: + - application/vnd.elasticsearch+json; compatible-with=8 + user-agent: + - elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0) + x-elastic-client-meta: + - es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7 + method: POST + uri: http://localhost:9200/continuous-tests-pods-*/_search?scroll=2m + response: + body: + string: '{"_scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDnF1ZXJ5VGhlbkZldGNoDBZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YgWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YwWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YoWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YkWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YsWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y0WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y4WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZMWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZAWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y8WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZEWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZIWVmdpQjdNT1RUM3VuSmxXekR6ZnREZw==","took":13,"timed_out":false,"_shards":{"total":12,"successful":12,"skipped":0,"failed":0},"hits":{"total":{"value":80036,"relation":"eq"},"max_score":null,"hits":[{"_index":"continuous-tests-pods-2024.02.08","_id":"lpQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.918606306Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"Internal: + Set CODEX_NAT: 10.244.4.178","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563918]},{"_index":"continuous-tests-pods-2024.02.08","_id":"l5Qmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.918876891Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"Run + Codex node","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563918]},{"_index":"continuous-tests-pods-2024.02.08","_id":"mJQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.927052796Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"NOT + 2024-02-08 23:56:03.926+00:00 Starting metrics HTTP server tid=1 + url=http://0.0.0.0:30006/metrics count=1","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563927]},{"_index":"continuous-tests-pods-2024.02.08","_id":"mZQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.934921297Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"WRN + 2024-02-08 23:56:03.926+00:00 Data directory has insecure permissions. Correcting + them. tid=1 data_dir=datadir3 current_permissions=0755 required_permissions=0700 + count=2","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563934]},{"_index":"continuous-tests-pods-2024.02.08","_id":"mpQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.934945015Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC + 2024-02-08 23:56:03.926+00:00 Data dir initialized topics=\"codex\" + tid=1 dir=datadir3 count=3","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563934]},{"_index":"continuous-tests-pods-2024.02.08","_id":"m5Qmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.934955237Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC + 2024-02-08 23:56:03.929+00:00 Repo dir initialized topics=\"codex\" + tid=1 dir=datadir3/repo count=4","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563934]},{"_index":"continuous-tests-pods-2024.02.08","_id":"nJQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.934960073Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"INF + 2024-02-08 23:56:03.929+00:00 Creating a private key and saving it tid=1 + count=5","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563934]}]}}' + headers: + Transfer-Encoding: + - chunked + X-elastic-product: + - Elasticsearch + content-type: + - application/vnd.elasticsearch+json;compatible-with=8 + status: + code: 200 + message: OK +- request: + body: '{"scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDnF1ZXJ5VGhlbkZldGNoDBZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YgWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YwWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YoWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YkWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YsWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y0WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y4WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZMWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZAWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y8WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZEWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZIWVmdpQjdNT1RUM3VuSmxXekR6ZnREZw==","scroll":"2m"}' + headers: + accept: + - application/vnd.elasticsearch+json; compatible-with=8 + connection: + - keep-alive + content-type: + - application/vnd.elasticsearch+json; compatible-with=8 + user-agent: + - elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0) + x-elastic-client-meta: + - es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7 + method: POST + uri: http://localhost:9200/_search/scroll + response: + body: + string: '{"_scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDnF1ZXJ5VGhlbkZldGNoDBZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YgWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YwWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YoWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YkWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YsWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y0WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y4WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZMWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZAWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y8WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZEWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZIWVmdpQjdNT1RUM3VuSmxXekR6ZnREZw==","took":3,"timed_out":false,"_shards":{"total":12,"successful":12,"skipped":0,"failed":0},"hits":{"total":{"value":80036,"relation":"eq"},"max_score":null,"hits":[{"_index":"continuous-tests-pods-2024.02.08","_id":"nZQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.950492498Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC + 2024-02-08 23:56:03.950+00:00 Updating announce record topics=\"codex + discovery\" tid=1 addrs=@[/ip4/0.0.0.0/tcp/8081] count=6","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563950]},{"_index":"continuous-tests-pods-2024.02.08","_id":"npQmi40BTe2pYqi1cfr7","_score":null,"_ignored":["message.keyword"],"_source":{"@timestamp":"2024-02-08T23:56:03.969109839Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC + 2024-02-08 23:56:03.968+00:00 Starting codex node topics=\"codex + node\" tid=1 config=\"(configFile: none(InputFile), logLevel: \\\"TRACE;warn:discv5,providers,manager,cache;warn:libp2p,multistream,switch,transport,tcptransport,semaphore,asyncstreamwrapper,lpstream,mplex,mplexchannel,noise,bufferstream,mplexcoder,secure,chronosstream,connection,connmanager,websock,ws-session,dialer,muxedupgrade,upgrade,identify\\\", + logFormat: auto, metricsEnabled: true, metricsAddress: 0.0.0.0, metricsPort: + 30006, dataDir: datadir3, profilerMaxMetrics: 100, cmd: noCommand, listenAddrs: + @[/ip4/0.0.0.0/tcp/8081], nat: 10.244.4.178, discoveryIp: 0.0.0.0, discoveryPort: + 8080, netPrivKeyFile: \\\"key\\\", bootstrapNodes: @[(envelope: (publicKey: + secp256k1 key (040b1beda7b6b56401c2c29554df74729e75b0906d0e8848dcc5dd04b28d2fcad4bd0b489cdc7a5a3a1c0e9ec0d0d3d8004c946da4baa1752fcb201feda1304d3b), + domain: \\\"libp2p-peer-record\\\", payloadType: @[3, 1], payload: @[10, 39, + 0, 37, 8, 2, 18, 33, 3, 11, 27, 237, 167, 182, 181, 100, 1, 194, 194, 149, + 84, 223, 116, 114, 158, 117, 176, 144, 109, 14, 136, 72, 220, 197, 221, 4, + 178, 141, 47, 202, 212, 16, 178, 210, 149, 174, 6, 26, 11, 10, 9, 4, 10, 244, + 2, 44, 145, 2, 31, 144], signature: 304402201C49248DA73B2055951C115F46301D8D2B72FAF83423CC45F69E9BED8F1E9679022076A752567FD7D7E7A262D90121C9B69DA9D76648728E23C1DAE6EDBB6B2A6811), + data: (peerId: 16Uiu2HAmDQMXzu725FEnd6pgzJr6iLJGt8TffyNK673VBArAMptT, seqNo: + 1707436338, addresses: @[(address: /ip4/10.244.2.44/udp/8080)]))], maxPeers: + 160, agentString: \\\"Codex\\\", apiBindAddress: \\\"0.0.0.0\\\", apiPort: + 30005, repoKind: fs, storageQuota: 62914560000\\''NByte, blockTtl: 16w3d17h46m39s, + blockMaintenanceInterval: 16w3d17h46m39s, blockMaintenanceNumberOfBlocks: + 10000, cacheSize: 0\\''NByte, persistence: false, ethProvider: \\\"ws://localhost:8545\\\", + ethAccount: none(Address), ethPrivateKey: none(string), marketplaceAddress: + none(Address), validator: false, validatorMaxSlots: 1000, simulateProofFailures: + 0, logFile: none(string))\" count=7","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563969]},{"_index":"continuous-tests-pods-2024.02.08","_id":"n5Qmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.969228477Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC + 2024-02-08 23:56:03.968+00:00 Starting repo topics=\"codex + repostore\" tid=1 count=8","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563969]},{"_index":"continuous-tests-pods-2024.02.08","_id":"oJQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.969273960Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC + 2024-02-08 23:56:03.969+00:00 Number of blocks in store at start topics=\"codex + repostore\" tid=1 total=0 count=9","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563969]},{"_index":"continuous-tests-pods-2024.02.08","_id":"oZQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.969452435Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"NOT + 2024-02-08 23:56:03.969+00:00 Current bytes used for cache quota topics=\"codex + repostore\" tid=1 bytes=0 count=10","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563969]},{"_index":"continuous-tests-pods-2024.02.08","_id":"opQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.969572555Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"NOT + 2024-02-08 23:56:03.969+00:00 Current bytes used for persist quota topics=\"codex + repostore\" tid=1 bytes=0 count=11","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563969]},{"_index":"continuous-tests-pods-2024.02.08","_id":"o5Qmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.969654771Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC + 2024-02-08 23:56:03.969+00:00 Timer starting: tid=1 + name=\"Unnamed Timer\" count=12","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563969]}]}}' + headers: + Transfer-Encoding: + - chunked + X-elastic-product: + - Elasticsearch + content-type: + - application/vnd.elasticsearch+json;compatible-with=8 + status: + code: 200 + message: OK +- request: + body: '{"scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDnF1ZXJ5VGhlbkZldGNoDBZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YgWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YwWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YoWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YkWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YsWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y0WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y4WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZMWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZAWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y8WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZEWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZIWVmdpQjdNT1RUM3VuSmxXekR6ZnREZw=="}' + headers: + accept: + - application/vnd.elasticsearch+json; compatible-with=8 + connection: + - keep-alive + content-type: + - application/vnd.elasticsearch+json; compatible-with=8 + user-agent: + - elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0) + x-elastic-client-meta: + - es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7 + method: DELETE + uri: http://localhost:9200/_search/scroll + response: + body: + string: '{"succeeded":true,"num_freed":12}' + headers: + X-elastic-product: + - Elasticsearch + content-length: + - '33' + content-type: + - application/vnd.elasticsearch+json;compatible-with=8 + status: + code: 200 + message: OK +version: 1 diff --git a/logtools/log/sources/input/tests/cassettes/test_elasticsearch_source/test_should_respect_fetching_limits_when_limit_smaller_than_batch_size.yaml b/logtools/log/sources/input/tests/cassettes/test_elasticsearch_source/test_should_respect_fetching_limits_when_limit_smaller_than_batch_size.yaml new file mode 100644 index 0000000..76b4b67 --- /dev/null +++ b/logtools/log/sources/input/tests/cassettes/test_elasticsearch_source/test_should_respect_fetching_limits_when_limit_smaller_than_batch_size.yaml @@ -0,0 +1,96 @@ +interactions: +- request: + body: '{"query":{"bool":{"filter":[{"terms":{"pod_name.keyword":["codex1-3-6c565dbd66-wxg49"]}},{"term":{"pod_labels.runid.keyword":"20240208-115030"}}]}},"size":10,"sort":[{"@timestamp":"asc"}]}' + headers: + accept: + - application/vnd.elasticsearch+json; compatible-with=8 + connection: + - keep-alive + content-type: + - application/vnd.elasticsearch+json; compatible-with=8 + user-agent: + - elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0) + x-elastic-client-meta: + - es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7 + method: POST + uri: http://localhost:9200/continuous-tests-pods-*/_search?scroll=2m + response: + body: + string: '{"_scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDnF1ZXJ5VGhlbkZldGNoDBZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7XwWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7X0WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7X4WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7X8WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YAWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YEWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YIWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YMWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YQWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YUWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YYWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YcWVmdpQjdNT1RUM3VuSmxXekR6ZnREZw==","took":10,"timed_out":false,"_shards":{"total":12,"successful":12,"skipped":0,"failed":0},"hits":{"total":{"value":80036,"relation":"eq"},"max_score":null,"hits":[{"_index":"continuous-tests-pods-2024.02.08","_id":"lpQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.918606306Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"Internal: + Set CODEX_NAT: 10.244.4.178","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563918]},{"_index":"continuous-tests-pods-2024.02.08","_id":"l5Qmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.918876891Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"Run + Codex node","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563918]},{"_index":"continuous-tests-pods-2024.02.08","_id":"mJQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.927052796Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"NOT + 2024-02-08 23:56:03.926+00:00 Starting metrics HTTP server tid=1 + url=http://0.0.0.0:30006/metrics count=1","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563927]},{"_index":"continuous-tests-pods-2024.02.08","_id":"mZQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.934921297Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"WRN + 2024-02-08 23:56:03.926+00:00 Data directory has insecure permissions. Correcting + them. tid=1 data_dir=datadir3 current_permissions=0755 required_permissions=0700 + count=2","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563934]},{"_index":"continuous-tests-pods-2024.02.08","_id":"mpQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.934945015Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC + 2024-02-08 23:56:03.926+00:00 Data dir initialized topics=\"codex\" + tid=1 dir=datadir3 count=3","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563934]},{"_index":"continuous-tests-pods-2024.02.08","_id":"m5Qmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.934955237Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC + 2024-02-08 23:56:03.929+00:00 Repo dir initialized topics=\"codex\" + tid=1 dir=datadir3/repo count=4","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563934]},{"_index":"continuous-tests-pods-2024.02.08","_id":"nJQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.934960073Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"INF + 2024-02-08 23:56:03.929+00:00 Creating a private key and saving it tid=1 + count=5","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563934]},{"_index":"continuous-tests-pods-2024.02.08","_id":"nZQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.950492498Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC + 2024-02-08 23:56:03.950+00:00 Updating announce record topics=\"codex + discovery\" tid=1 addrs=@[/ip4/0.0.0.0/tcp/8081] count=6","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563950]},{"_index":"continuous-tests-pods-2024.02.08","_id":"npQmi40BTe2pYqi1cfr7","_score":null,"_ignored":["message.keyword"],"_source":{"@timestamp":"2024-02-08T23:56:03.969109839Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC + 2024-02-08 23:56:03.968+00:00 Starting codex node topics=\"codex + node\" tid=1 config=\"(configFile: none(InputFile), logLevel: \\\"TRACE;warn:discv5,providers,manager,cache;warn:libp2p,multistream,switch,transport,tcptransport,semaphore,asyncstreamwrapper,lpstream,mplex,mplexchannel,noise,bufferstream,mplexcoder,secure,chronosstream,connection,connmanager,websock,ws-session,dialer,muxedupgrade,upgrade,identify\\\", + logFormat: auto, metricsEnabled: true, metricsAddress: 0.0.0.0, metricsPort: + 30006, dataDir: datadir3, profilerMaxMetrics: 100, cmd: noCommand, listenAddrs: + @[/ip4/0.0.0.0/tcp/8081], nat: 10.244.4.178, discoveryIp: 0.0.0.0, discoveryPort: + 8080, netPrivKeyFile: \\\"key\\\", bootstrapNodes: @[(envelope: (publicKey: + secp256k1 key (040b1beda7b6b56401c2c29554df74729e75b0906d0e8848dcc5dd04b28d2fcad4bd0b489cdc7a5a3a1c0e9ec0d0d3d8004c946da4baa1752fcb201feda1304d3b), + domain: \\\"libp2p-peer-record\\\", payloadType: @[3, 1], payload: @[10, 39, + 0, 37, 8, 2, 18, 33, 3, 11, 27, 237, 167, 182, 181, 100, 1, 194, 194, 149, + 84, 223, 116, 114, 158, 117, 176, 144, 109, 14, 136, 72, 220, 197, 221, 4, + 178, 141, 47, 202, 212, 16, 178, 210, 149, 174, 6, 26, 11, 10, 9, 4, 10, 244, + 2, 44, 145, 2, 31, 144], signature: 304402201C49248DA73B2055951C115F46301D8D2B72FAF83423CC45F69E9BED8F1E9679022076A752567FD7D7E7A262D90121C9B69DA9D76648728E23C1DAE6EDBB6B2A6811), + data: (peerId: 16Uiu2HAmDQMXzu725FEnd6pgzJr6iLJGt8TffyNK673VBArAMptT, seqNo: + 1707436338, addresses: @[(address: /ip4/10.244.2.44/udp/8080)]))], maxPeers: + 160, agentString: \\\"Codex\\\", apiBindAddress: \\\"0.0.0.0\\\", apiPort: + 30005, repoKind: fs, storageQuota: 62914560000\\''NByte, blockTtl: 16w3d17h46m39s, + blockMaintenanceInterval: 16w3d17h46m39s, blockMaintenanceNumberOfBlocks: + 10000, cacheSize: 0\\''NByte, persistence: false, ethProvider: \\\"ws://localhost:8545\\\", + ethAccount: none(Address), ethPrivateKey: none(string), marketplaceAddress: + none(Address), validator: false, validatorMaxSlots: 1000, simulateProofFailures: + 0, logFile: none(string))\" count=7","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563969]},{"_index":"continuous-tests-pods-2024.02.08","_id":"n5Qmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.969228477Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC + 2024-02-08 23:56:03.968+00:00 Starting repo topics=\"codex + repostore\" tid=1 count=8","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563969]}]}}' + headers: + Transfer-Encoding: + - chunked + X-elastic-product: + - Elasticsearch + content-type: + - application/vnd.elasticsearch+json;compatible-with=8 + status: + code: 200 + message: OK +- request: + body: '{"scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDnF1ZXJ5VGhlbkZldGNoDBZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7XwWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7X0WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7X4WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7X8WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YAWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YEWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YIWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YMWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YQWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YUWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YYWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YcWVmdpQjdNT1RUM3VuSmxXekR6ZnREZw=="}' + headers: + accept: + - application/vnd.elasticsearch+json; compatible-with=8 + connection: + - keep-alive + content-type: + - application/vnd.elasticsearch+json; compatible-with=8 + user-agent: + - elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0) + x-elastic-client-meta: + - es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7 + method: DELETE + uri: http://localhost:9200/_search/scroll + response: + body: + string: '{"succeeded":true,"num_freed":12}' + headers: + X-elastic-product: + - Elasticsearch + content-length: + - '33' + content-type: + - application/vnd.elasticsearch+json;compatible-with=8 + status: + code: 200 + message: OK +version: 1 diff --git a/logtools/log/sources/input/tests/test_elasticsearch_source.py b/logtools/log/sources/input/tests/test_elasticsearch_source.py index 32b2fb4..aef5c7c 100644 --- a/logtools/log/sources/input/tests/test_elasticsearch_source.py +++ b/logtools/log/sources/input/tests/test_elasticsearch_source.py @@ -28,6 +28,7 @@ def test_should_fetch_logs_by_date(): assert {line.location.pod_name for line in lines} == {'codex1-3-b558568cf-tvcsc', 'bootstrap-2-58b69484bc-88msf'} assert {line.location.run_id for line in lines} == {'20231109-101554'} + @pytest.mark.vcr def test_should_fetch_logs_when_no_dates_are_specified(): log = ElasticSearchSource( @@ -39,3 +40,30 @@ def test_should_fetch_logs_when_no_dates_are_specified(): next(log.__iter__()) except StopIteration: assert False, "Should have returned at least one log line" + + +@pytest.mark.vcr +def test_should_respect_fetching_limits_when_limit_smaller_than_batch_size(): + log = ElasticSearchSource( + run_id='20240208-115030', + pods={'codex1-3-6c565dbd66-wxg49'}, + limit=10 + ) + + lines = list(log) + assert len(lines) == 10 + assert log.page_fetch_counter == 1 + + +@pytest.mark.vcr +def test_should_respect_fetching_limits_when_limit_larger_than_batch_size(): + log = ElasticSearchSource( + run_id='20240208-115030', + pods={'codex1-3-6c565dbd66-wxg49'}, + es_batch_size=7, + limit=10 + ) + + lines = list(log) + assert len(lines) == 10 + assert log.page_fetch_counter == 2 diff --git a/pyproject.toml b/pyproject.toml index 49479cd..550a763 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "logtools" -version = "1.2.3" +version = "1.3.0" description = "" authors = ["gmega "] readme = "README.md"