codex-storage
/
logtools
mirror of https://github.com/codex-storage/logtools.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add log fetch limit

This commit is contained in:
gmega 2024-02-09 15:25:59 -03:00
parent d8ef2ca768
commit 8cc04e450e
No known key found for this signature in database
GPG Key ID: FFD8DAF00660270F
6 changed files with 284 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
logtools/cli/es_logs.py
View File

@ -14,10 +14,10 @@ from rich.console import Console
from rich.json import JSON
from rich.table import Table
from logtools import version_string
from logtools.cli.palettes import ColorMap
from logtools.log.sources.input.elastic_search_source import ElasticSearchSource
from logtools.resource.elastic_search_log_repo import ElasticSearchLogRepo
from logtools import version_string
class ResourceType(Enum):
@ -79,21 +79,23 @@ def get_logs(args, client: Elasticsearch):
)
elif resource == ResourceType.runs:
run = ElasticSearchLogRepo(client=client).test_run(test_run_id=args.test_run_id).test_run
get_pod_logs(set(run.pods), client, start_date=run.start, end_date=run.end)
get_pod_logs(set(run.pods), client, limit=args.limit, start_date=run.start, end_date=run.end)
def get_pod_logs(pods: Set[str],
client: Elasticsearch,
colored_output: bool = True,
limit: Optional[int] = None,
start_date: Optional[datetime] = None,
end_date: Optional[datetime] = None):
colors = ColorMap()
for line in ElasticSearchSource(
for i, line in enumerate(ElasticSearchSource(
pods=pods,
client=client,
start_date=start_date,
end_date=end_date,
):
limit=limit,
)):
output = f'[{line.location.pod_name}]: {line.raw}'
if colored_output:
output = f'{colors[line.location.pod_name]}{output}{Style.reset}'
@ -176,6 +178,7 @@ def _add_describe_cli(subparsers):
def _add_logs_cli(subparsers):
logs = subparsers.add_parser('logs', help='fetch pod logs')
logs.set_defaults(main=get_logs)
logs.add_argument('--limit', type=int, help='limit the number of log entries to fetch')
log_subparsers = logs.add_subparsers(title='resource type', dest='resource_type', required=True)
@ -184,11 +187,11 @@ def _add_logs_cli(subparsers):
pod_logs = log_subparsers.add_parser('pods', help='fetch logs for a pod')
pod_logs.add_argument('pods', nargs='+', help='pod names to fetch logs from')
pod_logs.add_argument('--from', dest='from_', type=tsparser.parse,
help='show entries from date/time (MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm), '
'treated as UTC if no timezone given', default=None)
help='show entries from date/time (MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm), '
'treated as UTC if no timezone given', default=None)
pod_logs.add_argument('--to', dest='to', type=tsparser.parse,
help='show entries until date/time (MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm), '
'treated as UTC if no timezone given', default=None)
help='show entries until date/time (MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm), '
'treated as UTC if no timezone given', default=None)
run_logs = log_subparsers.add_parser('runs', help='fetch logs for a test run')
run_logs.add_argument('test_run_id', help='run ID to fetch logs from')

19
logtools/log/sources/input/elastic_search_source.py
View File

@ -11,6 +11,7 @@ from logtools.log.utils import tree
logger = logging.getLogger(__name__)
INDEX_PREFIX = 'continuous-tests-pods'
ES_MAX_BATCH_SIZE = 10_000
@dataclass
@ -29,6 +30,8 @@ class ElasticSearchSource(LogSource[TimestampedLogLine[ElasticSearchLocation]]):
client: Optional[Elasticsearch] = None,
start_date: Optional[datetime] = None,
end_date: Optional[datetime] = None,
limit: Optional[int] = None,
es_batch_size=ES_MAX_BATCH_SIZE
):
if client is None:
logger.warning('No client provided, defaulting to localhost')
@ -40,6 +43,9 @@ class ElasticSearchSource(LogSource[TimestampedLogLine[ElasticSearchLocation]]):
self.client = client
self.start_date = start_date
self.end_date = end_date
self.limit = limit
self.es_batch_size = es_batch_size
self.page_fetch_counter = 0
def __iter__(self) -> Iterator[TimestampedLogLine[ElasticSearchLocation]]:
for index in self._indices():
@ -88,8 +94,11 @@ class ElasticSearchSource(LogSource[TimestampedLogLine[ElasticSearchLocation]]):
return query
def _run_scan(self, query: Dict[str, Any], index: str):
# the search type stub does not contain the body argument for some reason so we disable typing here.
initial = self.client.search(index=index, body=query, size=10_000, scroll='2m') # type: ignore
remaining = self.limit if self.limit is not None else float('inf')
# XXX the search type stub does not contain the body argument for some reason so we disable typing here.
initial = self.client.search(index=index, body=query, size=min(remaining, self.es_batch_size),
scroll='2m') # type: ignore
self.page_fetch_counter += 1
scroll_id = initial['_scroll_id']
results = initial
@ -97,12 +106,16 @@ class ElasticSearchSource(LogSource[TimestampedLogLine[ElasticSearchLocation]]):
while True:
documents = results['hits']['hits']
if not documents:
break
return
for doc in documents:
yield doc
remaining -= 1
if remaining <= 0:
return
results = self.client.scroll(scroll_id=scroll_id, scroll='2m')
self.page_fetch_counter += 1
finally:
self.client.clear_scroll(scroll_id=scroll_id)

132
logtools/log/sources/input/tests/cassettes/test_elasticsearch_source/test_should_respect_fetching_limits_when_limit_larger_than_batch_size.yaml Normal file
View File

@ -0,0 +1,132 @@
interactions:
- request:
body: '{"query":{"bool":{"filter":[{"terms":{"pod_name.keyword":["codex1-3-6c565dbd66-wxg49"]}},{"term":{"pod_labels.runid.keyword":"20240208-115030"}}]}},"size":7,"sort":[{"@timestamp":"asc"}]}'
headers:
accept:
- application/vnd.elasticsearch+json; compatible-with=8
connection:
- keep-alive
content-type:
- application/vnd.elasticsearch+json; compatible-with=8
user-agent:
- elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0)
x-elastic-client-meta:
- es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7
method: POST
uri: http://localhost:9200/continuous-tests-pods-*/_search?scroll=2m
response:
body:
string: '{"_scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDnF1ZXJ5VGhlbkZldGNoDBZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YgWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YwWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YoWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YkWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YsWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y0WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y4WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZMWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZAWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y8WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZEWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZIWVmdpQjdNT1RUM3VuSmxXekR6ZnREZw==","took":13,"timed_out":false,"_shards":{"total":12,"successful":12,"skipped":0,"failed":0},"hits":{"total":{"value":80036,"relation":"eq"},"max_score":null,"hits":[{"_index":"continuous-tests-pods-2024.02.08","_id":"lpQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.918606306Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"Internal:
Set CODEX_NAT: 10.244.4.178","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563918]},{"_index":"continuous-tests-pods-2024.02.08","_id":"l5Qmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.918876891Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"Run
Codex node","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563918]},{"_index":"continuous-tests-pods-2024.02.08","_id":"mJQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.927052796Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"NOT
2024-02-08 23:56:03.926+00:00 Starting metrics HTTP server tid=1
url=http://0.0.0.0:30006/metrics count=1","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563927]},{"_index":"continuous-tests-pods-2024.02.08","_id":"mZQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.934921297Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"WRN
2024-02-08 23:56:03.926+00:00 Data directory has insecure permissions. Correcting
them. tid=1 data_dir=datadir3 current_permissions=0755 required_permissions=0700
count=2","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563934]},{"_index":"continuous-tests-pods-2024.02.08","_id":"mpQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.934945015Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC
2024-02-08 23:56:03.926+00:00 Data dir initialized topics=\"codex\"
tid=1 dir=datadir3 count=3","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563934]},{"_index":"continuous-tests-pods-2024.02.08","_id":"m5Qmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.934955237Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC
2024-02-08 23:56:03.929+00:00 Repo dir initialized topics=\"codex\"
tid=1 dir=datadir3/repo count=4","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563934]},{"_index":"continuous-tests-pods-2024.02.08","_id":"nJQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.934960073Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"INF
2024-02-08 23:56:03.929+00:00 Creating a private key and saving it tid=1
count=5","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563934]}]}}'
headers:
Transfer-Encoding:
- chunked
X-elastic-product:
- Elasticsearch
content-type:
- application/vnd.elasticsearch+json;compatible-with=8
status:
code: 200
message: OK
- request:
body: '{"scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDnF1ZXJ5VGhlbkZldGNoDBZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YgWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YwWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YoWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YkWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YsWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y0WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y4WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZMWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZAWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y8WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZEWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZIWVmdpQjdNT1RUM3VuSmxXekR6ZnREZw==","scroll":"2m"}'
headers:
accept:
- application/vnd.elasticsearch+json; compatible-with=8
connection:
- keep-alive
content-type:
- application/vnd.elasticsearch+json; compatible-with=8
user-agent:
- elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0)
x-elastic-client-meta:
- es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7
method: POST
uri: http://localhost:9200/_search/scroll
response:
body:
string: '{"_scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDnF1ZXJ5VGhlbkZldGNoDBZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YgWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YwWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YoWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YkWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YsWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y0WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y4WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZMWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZAWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y8WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZEWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZIWVmdpQjdNT1RUM3VuSmxXekR6ZnREZw==","took":3,"timed_out":false,"_shards":{"total":12,"successful":12,"skipped":0,"failed":0},"hits":{"total":{"value":80036,"relation":"eq"},"max_score":null,"hits":[{"_index":"continuous-tests-pods-2024.02.08","_id":"nZQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.950492498Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC
2024-02-08 23:56:03.950+00:00 Updating announce record topics=\"codex
discovery\" tid=1 addrs=@[/ip4/0.0.0.0/tcp/8081] count=6","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563950]},{"_index":"continuous-tests-pods-2024.02.08","_id":"npQmi40BTe2pYqi1cfr7","_score":null,"_ignored":["message.keyword"],"_source":{"@timestamp":"2024-02-08T23:56:03.969109839Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC
2024-02-08 23:56:03.968+00:00 Starting codex node topics=\"codex
node\" tid=1 config=\"(configFile: none(InputFile), logLevel: \\\"TRACE;warn:discv5,providers,manager,cache;warn:libp2p,multistream,switch,transport,tcptransport,semaphore,asyncstreamwrapper,lpstream,mplex,mplexchannel,noise,bufferstream,mplexcoder,secure,chronosstream,connection,connmanager,websock,ws-session,dialer,muxedupgrade,upgrade,identify\\\",
logFormat: auto, metricsEnabled: true, metricsAddress: 0.0.0.0, metricsPort:
30006, dataDir: datadir3, profilerMaxMetrics: 100, cmd: noCommand, listenAddrs:
@[/ip4/0.0.0.0/tcp/8081], nat: 10.244.4.178, discoveryIp: 0.0.0.0, discoveryPort:
8080, netPrivKeyFile: \\\"key\\\", bootstrapNodes: @[(envelope: (publicKey:
secp256k1 key (040b1beda7b6b56401c2c29554df74729e75b0906d0e8848dcc5dd04b28d2fcad4bd0b489cdc7a5a3a1c0e9ec0d0d3d8004c946da4baa1752fcb201feda1304d3b),
domain: \\\"libp2p-peer-record\\\", payloadType: @[3, 1], payload: @[10, 39,
0, 37, 8, 2, 18, 33, 3, 11, 27, 237, 167, 182, 181, 100, 1, 194, 194, 149,
84, 223, 116, 114, 158, 117, 176, 144, 109, 14, 136, 72, 220, 197, 221, 4,
178, 141, 47, 202, 212, 16, 178, 210, 149, 174, 6, 26, 11, 10, 9, 4, 10, 244,
2, 44, 145, 2, 31, 144], signature: 304402201C49248DA73B2055951C115F46301D8D2B72FAF83423CC45F69E9BED8F1E9679022076A752567FD7D7E7A262D90121C9B69DA9D76648728E23C1DAE6EDBB6B2A6811),
data: (peerId: 16Uiu2HAmDQMXzu725FEnd6pgzJr6iLJGt8TffyNK673VBArAMptT, seqNo:
1707436338, addresses: @[(address: /ip4/10.244.2.44/udp/8080)]))], maxPeers:
160, agentString: \\\"Codex\\\", apiBindAddress: \\\"0.0.0.0\\\", apiPort:
30005, repoKind: fs, storageQuota: 62914560000\\''NByte, blockTtl: 16w3d17h46m39s,
blockMaintenanceInterval: 16w3d17h46m39s, blockMaintenanceNumberOfBlocks:
10000, cacheSize: 0\\''NByte, persistence: false, ethProvider: \\\"ws://localhost:8545\\\",
ethAccount: none(Address), ethPrivateKey: none(string), marketplaceAddress:
none(Address), validator: false, validatorMaxSlots: 1000, simulateProofFailures:
0, logFile: none(string))\" count=7","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563969]},{"_index":"continuous-tests-pods-2024.02.08","_id":"n5Qmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.969228477Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC
2024-02-08 23:56:03.968+00:00 Starting repo topics=\"codex
repostore\" tid=1 count=8","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563969]},{"_index":"continuous-tests-pods-2024.02.08","_id":"oJQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.969273960Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC
2024-02-08 23:56:03.969+00:00 Number of blocks in store at start topics=\"codex
repostore\" tid=1 total=0 count=9","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563969]},{"_index":"continuous-tests-pods-2024.02.08","_id":"oZQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.969452435Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"NOT
2024-02-08 23:56:03.969+00:00 Current bytes used for cache quota topics=\"codex
repostore\" tid=1 bytes=0 count=10","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563969]},{"_index":"continuous-tests-pods-2024.02.08","_id":"opQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.969572555Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"NOT
2024-02-08 23:56:03.969+00:00 Current bytes used for persist quota topics=\"codex
repostore\" tid=1 bytes=0 count=11","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563969]},{"_index":"continuous-tests-pods-2024.02.08","_id":"o5Qmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.969654771Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC
2024-02-08 23:56:03.969+00:00 Timer starting: tid=1
name=\"Unnamed Timer\" count=12","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563969]}]}}'
headers:
Transfer-Encoding:
- chunked
X-elastic-product:
- Elasticsearch
content-type:
- application/vnd.elasticsearch+json;compatible-with=8
status:
code: 200
message: OK
- request:
body: '{"scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDnF1ZXJ5VGhlbkZldGNoDBZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YgWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YwWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YoWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YkWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YsWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y0WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y4WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZMWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZAWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y8WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZEWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZIWVmdpQjdNT1RUM3VuSmxXekR6ZnREZw=="}'
headers:
accept:
- application/vnd.elasticsearch+json; compatible-with=8
connection:
- keep-alive
content-type:
- application/vnd.elasticsearch+json; compatible-with=8
user-agent:
- elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0)
x-elastic-client-meta:
- es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7
method: DELETE
uri: http://localhost:9200/_search/scroll
response:
body:
string: '{"succeeded":true,"num_freed":12}'
headers:
X-elastic-product:
- Elasticsearch
content-length:
- '33'
content-type:
- application/vnd.elasticsearch+json;compatible-with=8
status:
code: 200
message: OK
version: 1

96
logtools/log/sources/input/tests/cassettes/test_elasticsearch_source/test_should_respect_fetching_limits_when_limit_smaller_than_batch_size.yaml Normal file
View File

@ -0,0 +1,96 @@
interactions:
- request:
body: '{"query":{"bool":{"filter":[{"terms":{"pod_name.keyword":["codex1-3-6c565dbd66-wxg49"]}},{"term":{"pod_labels.runid.keyword":"20240208-115030"}}]}},"size":10,"sort":[{"@timestamp":"asc"}]}'
headers:
accept:
- application/vnd.elasticsearch+json; compatible-with=8
connection:
- keep-alive
content-type:
- application/vnd.elasticsearch+json; compatible-with=8
user-agent:
- elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0)
x-elastic-client-meta:
- es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7
method: POST
uri: http://localhost:9200/continuous-tests-pods-*/_search?scroll=2m
response:
body:
string: '{"_scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDnF1ZXJ5VGhlbkZldGNoDBZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7XwWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7X0WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7X4WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7X8WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YAWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YEWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YIWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YMWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YQWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YUWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YYWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YcWVmdpQjdNT1RUM3VuSmxXekR6ZnREZw==","took":10,"timed_out":false,"_shards":{"total":12,"successful":12,"skipped":0,"failed":0},"hits":{"total":{"value":80036,"relation":"eq"},"max_score":null,"hits":[{"_index":"continuous-tests-pods-2024.02.08","_id":"lpQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.918606306Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"Internal:
Set CODEX_NAT: 10.244.4.178","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563918]},{"_index":"continuous-tests-pods-2024.02.08","_id":"l5Qmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.918876891Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"Run
Codex node","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563918]},{"_index":"continuous-tests-pods-2024.02.08","_id":"mJQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.927052796Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"NOT
2024-02-08 23:56:03.926+00:00 Starting metrics HTTP server tid=1
url=http://0.0.0.0:30006/metrics count=1","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563927]},{"_index":"continuous-tests-pods-2024.02.08","_id":"mZQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.934921297Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"WRN
2024-02-08 23:56:03.926+00:00 Data directory has insecure permissions. Correcting
them. tid=1 data_dir=datadir3 current_permissions=0755 required_permissions=0700
count=2","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563934]},{"_index":"continuous-tests-pods-2024.02.08","_id":"mpQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.934945015Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC
2024-02-08 23:56:03.926+00:00 Data dir initialized topics=\"codex\"
tid=1 dir=datadir3 count=3","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563934]},{"_index":"continuous-tests-pods-2024.02.08","_id":"m5Qmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.934955237Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC
2024-02-08 23:56:03.929+00:00 Repo dir initialized topics=\"codex\"
tid=1 dir=datadir3/repo count=4","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563934]},{"_index":"continuous-tests-pods-2024.02.08","_id":"nJQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.934960073Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"INF
2024-02-08 23:56:03.929+00:00 Creating a private key and saving it tid=1
count=5","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563934]},{"_index":"continuous-tests-pods-2024.02.08","_id":"nZQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.950492498Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC
2024-02-08 23:56:03.950+00:00 Updating announce record topics=\"codex
discovery\" tid=1 addrs=@[/ip4/0.0.0.0/tcp/8081] count=6","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563950]},{"_index":"continuous-tests-pods-2024.02.08","_id":"npQmi40BTe2pYqi1cfr7","_score":null,"_ignored":["message.keyword"],"_source":{"@timestamp":"2024-02-08T23:56:03.969109839Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC
2024-02-08 23:56:03.968+00:00 Starting codex node topics=\"codex
node\" tid=1 config=\"(configFile: none(InputFile), logLevel: \\\"TRACE;warn:discv5,providers,manager,cache;warn:libp2p,multistream,switch,transport,tcptransport,semaphore,asyncstreamwrapper,lpstream,mplex,mplexchannel,noise,bufferstream,mplexcoder,secure,chronosstream,connection,connmanager,websock,ws-session,dialer,muxedupgrade,upgrade,identify\\\",
logFormat: auto, metricsEnabled: true, metricsAddress: 0.0.0.0, metricsPort:
30006, dataDir: datadir3, profilerMaxMetrics: 100, cmd: noCommand, listenAddrs:
@[/ip4/0.0.0.0/tcp/8081], nat: 10.244.4.178, discoveryIp: 0.0.0.0, discoveryPort:
8080, netPrivKeyFile: \\\"key\\\", bootstrapNodes: @[(envelope: (publicKey:
secp256k1 key (040b1beda7b6b56401c2c29554df74729e75b0906d0e8848dcc5dd04b28d2fcad4bd0b489cdc7a5a3a1c0e9ec0d0d3d8004c946da4baa1752fcb201feda1304d3b),
domain: \\\"libp2p-peer-record\\\", payloadType: @[3, 1], payload: @[10, 39,
0, 37, 8, 2, 18, 33, 3, 11, 27, 237, 167, 182, 181, 100, 1, 194, 194, 149,
84, 223, 116, 114, 158, 117, 176, 144, 109, 14, 136, 72, 220, 197, 221, 4,
178, 141, 47, 202, 212, 16, 178, 210, 149, 174, 6, 26, 11, 10, 9, 4, 10, 244,
2, 44, 145, 2, 31, 144], signature: 304402201C49248DA73B2055951C115F46301D8D2B72FAF83423CC45F69E9BED8F1E9679022076A752567FD7D7E7A262D90121C9B69DA9D76648728E23C1DAE6EDBB6B2A6811),
data: (peerId: 16Uiu2HAmDQMXzu725FEnd6pgzJr6iLJGt8TffyNK673VBArAMptT, seqNo:
1707436338, addresses: @[(address: /ip4/10.244.2.44/udp/8080)]))], maxPeers:
160, agentString: \\\"Codex\\\", apiBindAddress: \\\"0.0.0.0\\\", apiPort:
30005, repoKind: fs, storageQuota: 62914560000\\''NByte, blockTtl: 16w3d17h46m39s,
blockMaintenanceInterval: 16w3d17h46m39s, blockMaintenanceNumberOfBlocks:
10000, cacheSize: 0\\''NByte, persistence: false, ethProvider: \\\"ws://localhost:8545\\\",
ethAccount: none(Address), ethPrivateKey: none(string), marketplaceAddress:
none(Address), validator: false, validatorMaxSlots: 1000, simulateProofFailures:
0, logFile: none(string))\" count=7","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563969]},{"_index":"continuous-tests-pods-2024.02.08","_id":"n5Qmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.969228477Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC
2024-02-08 23:56:03.968+00:00 Starting repo topics=\"codex
repostore\" tid=1 count=8","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563969]}]}}'
headers:
Transfer-Encoding:
- chunked
X-elastic-product:
- Elasticsearch
content-type:
- application/vnd.elasticsearch+json;compatible-with=8
status:
code: 200
message: OK
- request:
body: '{"scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDnF1ZXJ5VGhlbkZldGNoDBZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7XwWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7X0WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7X4WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7X8WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YAWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YEWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YIWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YMWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YQWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YUWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YYWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YcWVmdpQjdNT1RUM3VuSmxXekR6ZnREZw=="}'
headers:
accept:
- application/vnd.elasticsearch+json; compatible-with=8
connection:
- keep-alive
content-type:
- application/vnd.elasticsearch+json; compatible-with=8
user-agent:
- elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0)
x-elastic-client-meta:
- es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7
method: DELETE
uri: http://localhost:9200/_search/scroll
response:
body:
string: '{"succeeded":true,"num_freed":12}'
headers:
X-elastic-product:
- Elasticsearch
content-length:
- '33'
content-type:
- application/vnd.elasticsearch+json;compatible-with=8
status:
code: 200
message: OK
version: 1

28
logtools/log/sources/input/tests/test_elasticsearch_source.py
View File

@ -28,6 +28,7 @@ def test_should_fetch_logs_by_date():
assert {line.location.pod_name for line in lines} == {'codex1-3-b558568cf-tvcsc', 'bootstrap-2-58b69484bc-88msf'}
assert {line.location.run_id for line in lines} == {'20231109-101554'}
@pytest.mark.vcr
def test_should_fetch_logs_when_no_dates_are_specified():
log = ElasticSearchSource(
@ -39,3 +40,30 @@ def test_should_fetch_logs_when_no_dates_are_specified():
next(log.__iter__())
except StopIteration:
assert False, "Should have returned at least one log line"
@pytest.mark.vcr
def test_should_respect_fetching_limits_when_limit_smaller_than_batch_size():
log = ElasticSearchSource(
run_id='20240208-115030',
pods={'codex1-3-6c565dbd66-wxg49'},
limit=10
)
lines = list(log)
assert len(lines) == 10
assert log.page_fetch_counter == 1
@pytest.mark.vcr
def test_should_respect_fetching_limits_when_limit_larger_than_batch_size():
log = ElasticSearchSource(
run_id='20240208-115030',
pods={'codex1-3-6c565dbd66-wxg49'},
es_batch_size=7,
limit=10
)
lines = list(log)
assert len(lines) == 10
assert log.page_fetch_counter == 2

2
pyproject.toml
View File

@ -1,6 +1,6 @@
[tool.poetry]
name = "logtools"
version = "1.2.3"
version = "1.3.0"
description = ""
authors = ["gmega <giuliano@status.im>"]
readme = "README.md"