allow adding extra pods when downloading log runs

logtools/__init__.py

@@ -0,0 +1,10 @@
+from importlib import metadata
+
+try:
+    __version__ = metadata.version(__package__)
+except metadata.PackageNotFoundError:
+    __version__ = '<<unknown>>'
+
+version_string = f'logtools version {__version__}'
+
+del metadata

logtools/cli/es_logs.py

@@ -1,75 +1,45 @@
-import datetime
-import math
+import dataclasses
 import os
-import textwrap
 from argparse import ArgumentParser
-from datetime import timedelta
+from datetime import timedelta, datetime
 from enum import Enum
-from typing import List, Iterable
+from json import JSONEncoder
+from typing import List, Iterable, Any, Optional, Set
 
 from colored import Style
 from dateutil import parser as tsparser
 from elasticsearch import Elasticsearch
-from prettytable import PrettyTable
+from rich import json
+from rich.console import Console
+from rich.json import JSON
+from rich.table import Table
 
+from logtools import version_string
 from logtools.cli.palettes import ColorMap
-from logtools.log.sources.input.elastic_search.elastic_search_log_repo import ElasticSearchLogRepo
-from logtools.log.sources.input.elastic_search.elastic_search_source import ElasticSearchSource
+from logtools.log.sources.input.elastic_search_source import ElasticSearchSource
+from logtools.resource.elastic_search_log_repo import ElasticSearchLogRepo
 
 
 class ResourceType(Enum):
     pods = 'pods'
     namespaces = 'namespaces'
+    runs = 'runs'
 
 
-GETTERS = {
+RESOURCE_GETTERS = {
     ResourceType.pods: lambda repo, args: repo.pods(prefix=args.prefix, run_id=args.run_id),
-    ResourceType.namespaces: lambda repo, args: repo.namespaces(prefix=args.prefix)
+    ResourceType.namespaces: lambda repo, args: repo.namespaces(prefix=args.prefix),
+    ResourceType.runs: lambda repo, args: repo.test_runs(run_id=args.run_id, failed_only=args.failed_only),
 }
 
-
-def format_table(objects: List) -> str:
-    tbl = PrettyTable()
-    wrapper = None
-
-    for obj in objects:
-        if not tbl.field_names:
-            tbl.field_names = obj.__annotations__.keys()
-            wrapper = textwrap.TextWrapper(width=math.ceil(os.get_terminal_size().columns / len(tbl.field_names)),
-                                           break_long_words=False)
-
-        tbl.add_row([wrapper.fill(_format_field(getattr(obj, field))) for field in tbl.field_names])
-
-    return tbl.get_string()
-
-
-def _format_field(field: str | Iterable[object]):
-    if isinstance(field, str):
-        return field
-    return ', '.join([str(item) for item in field])
-
-
-def get_object(args, client: Elasticsearch):
-    repo = ElasticSearchLogRepo(client=client)
-    print(format_table(GETTERS[ResourceType[args.resource_type]](repo, args)))
-
-
-def get_logs(args, client: Elasticsearch):
-    colors = ColorMap()
-    for line in ElasticSearchSource(
-            pods=args.pods,
-            client=client,
-            start_date=args.from_,
-            end_date=args.to,
-    ):
-        output = f'[{line.location.pod_name}]: {line.raw}'
-        if not args.no_color:
-            output = f'{colors[line.location.pod_name]}{output}{Style.reset}'
-        print(output)
+RESOURCE_DESCRIBERS = {
+    ResourceType.runs: lambda repo, args: repo.test_run(test_run_id=args.test_run_id),
+}
 
 
 def main():
     parser = ArgumentParser()
+    parser.add_argument('--version', action='version', version=version_string)
     parser.add_argument(
         '--es-host',
         help='ElasticSearch URL (defaults to http://localhost:9200)',
@@ -77,40 +47,159 @@ def main():
     )
 
     subparsers = parser.add_subparsers(title='Command', required=True)
-
-    get = subparsers.add_parser('get', help='Display existing resources')
-    get.add_argument('--from', type=tsparser.parse,
-                     help='Show resources present in log messages starting at the given date '
-                          '(MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm). Defaults to 7 days ago.',
-                     default=(datetime.datetime.today() - timedelta(days=7)).date())
-    get.set_defaults(main=get_object)
-
-    get_subparsers = get.add_subparsers(title='Resource type', dest='resource_type', required=True)
-    get_pods = get_subparsers.add_parser('pods', help='Display existing pods')
-    get_pods.add_argument('--prefix', help='Filter pods by prefix')
-    get_pods.add_argument('--run-id', help='Show pods for a given run', required=True)
-
-    get_namespaces = get_subparsers.add_parser('namespaces', help='Display existing namespaces')
-    get_namespaces.add_argument('--prefix', help='Filter namespaces by prefix')
-
-    logs = subparsers.add_parser('logs', help='Fetch pod logs')
-    logs.set_defaults(main=get_logs)
-
-    logs.add_argument('--pods', nargs='+', help='Pods to fetch logs for', required=True)
-    logs.add_argument('--from', dest='from_', type=tsparser.parse,
-                      help='Show entries from date/time (MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm), '
-                           'treated as UTC if no timezone given', default=None)
-    logs.add_argument('--to', dest='to', type=tsparser.parse,
-                      help='Show entries until date/time (MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm), '
-                           'treated as UTC if no timezone given', default=None)
-    logs.add_argument('--no-color', dest='no_color', action='store_true', help='Disable colored output')
+    _add_get_cli(subparsers)
+    _add_describe_cli(subparsers)
+    _add_logs_cli(subparsers)
 
     args = parser.parse_args()
-
     client = Elasticsearch(args.es_host, request_timeout=60)
-
     args.main(args, client)
 
 
+def get_object(args, client: Elasticsearch):
+    repo = ElasticSearchLogRepo(client=client)
+    Console().print(format_table(RESOURCE_GETTERS[ResourceType[args.resource_type]](repo, args)))
+
+
+def describe_object(args, client: Elasticsearch):
+    repo = ElasticSearchLogRepo(client=client)
+    Console().print(format_json([RESOURCE_DESCRIBERS[ResourceType[args.resource_type]](repo, args)]))
+
+
+# FIXME this is starting to get too complex to be here.
+def get_logs(args, client: Elasticsearch):
+    resource = ResourceType[args.resource_type]
+    if resource == ResourceType.pods:
+        get_pod_logs(
+            pods=args.pods,
+            client=client,
+            colored_output=not args.no_color,
+            start_date=args.from_,
+            end_date=args.to,
+            limit=args.limit,
+        )
+    elif resource == ResourceType.runs:
+        run = ElasticSearchLogRepo(client=client).test_run(test_run_id=args.test_run_id).test_run
+        get_pod_logs(set(run.pods).union(set(args.additional_pods)),
+                     client, limit=args.limit, start_date=run.start, end_date=run.end)
+
+
+def get_pod_logs(pods: Set[str],
+                 client: Elasticsearch,
+                 colored_output: bool = True,
+                 limit: Optional[int] = None,
+                 start_date: Optional[datetime] = None,
+                 end_date: Optional[datetime] = None):
+    colors = ColorMap()
+    for i, line in enumerate(ElasticSearchSource(
+            pods=pods,
+            client=client,
+            start_date=start_date,
+            end_date=end_date,
+            limit=limit,
+    )):
+        output = f'[{line.location.pod_name}]: {line.raw}'
+        if colored_output:
+            output = f'{colors[line.location.pod_name]}{output}{Style.reset}'
+        print(output)
+
+
+def format_table(objects: List, title: str = 'Results') -> Table:
+    tbl = Table(title=title)
+    field_names = None
+
+    for obj in objects:
+        if field_names is None:
+            field_names = obj.__annotations__.keys()
+            for field_name in field_names:
+                tbl.add_column(field_name, justify='left')
+
+        tbl.add_row(*[_format_field(getattr(obj, field)) for field in field_names])
+
+    return tbl
+
+
+def format_json(obj: Any) -> JSON:
+    # For now, this is rather rudimentary.
+    class DataclassEncoder(JSONEncoder):
+        def default(self, o):
+            if dataclasses.is_dataclass(o):
+                return dataclasses.asdict(o)
+            elif isinstance(o, datetime):
+                return o.isoformat()
+            elif isinstance(o, Enum):
+                return o.value
+            return super().default(o)
+
+    return JSON(json.dumps(obj, cls=DataclassEncoder))
+
+
+def _format_field(field: Any):
+    if isinstance(field, str):
+        return field
+    elif isinstance(field, Iterable):
+        return ', '.join([_format_field(item) for item in field])
+    return str(field)
+
+
+def _add_get_cli(subparsers):
+    get = subparsers.add_parser('get', help='display existing resources')
+    get.add_argument('--from', type=tsparser.parse,
+                     help='show resources present in log messages starting at the given date '
+                          '(mm-dd-yyyy, or mm-dd-yyyy hh:mm:ss.mmmmmm). defaults to 7 days ago.',
+                     default=(datetime.today() - timedelta(days=7)).date())
+    get.set_defaults(main=get_object)
+
+    get_subparsers = get.add_subparsers(title='resource type', dest='resource_type', required=True)
+    get_pods = get_subparsers.add_parser('pods', help='display existing pods')
+    get_pods.add_argument('--prefix', help='filter pods by prefix')
+    get_pods.add_argument('run_id', help='show pods for a given run')
+
+    get_namespaces = get_subparsers.add_parser('namespaces', help='display existing namespaces')
+    get_namespaces.add_argument('--prefix', help='filter namespaces by prefix')
+
+    get_namespaces = get_subparsers.add_parser('runs', help='display current test runs')
+    get_namespaces.add_argument('run_id', help='show test runs for the given run id')
+    get_namespaces.add_argument('--failed-only', action='store_true', help='show only failed test runs')
+    get_namespaces.add_argument('--from', type=tsparser.parse,
+                                help='show test runs starting at the given date '
+                                     '(mm-dd-yyyy, or mm-dd-yyyy hh:mm:ss.mmmmmm). defaults to 7 days ago.',
+                                default=(datetime.today() - timedelta(days=7)).date())
+
+
+def _add_describe_cli(subparsers):
+    describe = subparsers.add_parser('describe', help='describe a resource')
+    describe.set_defaults(main=describe_object)
+
+    describe_subparsers = describe.add_subparsers(title='resource type', dest='resource_type', required=True)
+    describe_runs = describe_subparsers.add_parser('runs', help='describe a test run')
+    describe_runs.add_argument('test_run_id', help='show test run details')
+    describe_runs.set_defaults(main=describe_object)
+
+
+def _add_logs_cli(subparsers):
+    logs = subparsers.add_parser('logs', help='fetch pod logs')
+    logs.set_defaults(main=get_logs)
+    logs.add_argument('--limit', type=int, help='limit the number of log entries to fetch')
+
+    log_subparsers = logs.add_subparsers(title='resource type', dest='resource_type', required=True)
+
+    logs.add_argument('--no-color', dest='no_color', action='store_true', help='disable colored output')
+
+    pod_logs = log_subparsers.add_parser('pods', help='fetch logs for a pod')
+    pod_logs.add_argument('pods', nargs='+', help='pod names to fetch logs from')
+    pod_logs.add_argument('--from', dest='from_', type=tsparser.parse,
+                          help='show entries from date/time (MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm), '
+                               'treated as UTC if no timezone given', default=None)
+    pod_logs.add_argument('--to', dest='to', type=tsparser.parse,
+                          help='show entries until date/time (MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm), '
+                               'treated as UTC if no timezone given', default=None)
+
+    run_logs = log_subparsers.add_parser('runs', help='fetch logs for a test run')
+    run_logs.add_argument('test_run_id', help='run ID to fetch logs from')
+    run_logs.add_argument('--additional-pods', nargs='+', help='in addition to the pods in the test run, also '
+                                                               'fetches logs from these pods')
+
+
 if __name__ == '__main__':
     main()

logtools/cli/merge.py

@@ -10,6 +10,7 @@ import pytz
 from colored import Fore, Style
 from dateutil import parser as tsparser
 
+from logtools import version_string
 from logtools.log.sources.input.file_log_source import FileLogSource
 from logtools.log.sources.parse.chronicles_raw_source import ChroniclesRawSource
 from logtools.log.sources.transform.filtered_source import FilteredSource, timestamp_range
@@ -21,7 +22,7 @@ def merge(args):
     names = _assign_aliases(args)
     palette = _assign_colors(names)
 
-    logs = MergedSource(*[
+    parts = [
         OrderedSource(
             FilteredSource(
                 ChroniclesRawSource(
@@ -31,7 +32,10 @@ def merge(args):
             )
         )
         for path in args.files
-    ])
+    ]
+
+    # If we only have one source, then no need to actually do a merge.
+    logs = MergedSource(*parts) if len(parts) > 1 else parts[0]
 
     for line in logs:
         log_id = names[line.location.path.name]
@@ -77,6 +81,7 @@ def _ensure_utc(ts: datetime) -> datetime:
 def main():
     parser = argparse.ArgumentParser(
         description='Merges Chronicles logs chronologically and outputs colored, interleaved content.')
+    parser.add_argument('--version', action='version', version=version_string)
 
     parser.add_argument("files", nargs="+", help='Log files to merge.', type=Path)
     parser.add_argument('--aliases', nargs="*",

logtools/cli/to_csv.py

@@ -5,6 +5,7 @@ from argparse import ArgumentParser
 from csv import DictWriter
 from pathlib import Path
 
+from logtools import version_string
 from logtools.cli.utils import kv_pair
 from logtools.log.sources.input.file_log_source import FileLogSource
 from logtools.log.sources.parse.chronicles_raw_source import ChroniclesRawSource
@@ -37,6 +38,7 @@ def to_csv(args):
 
 def main():
     argparse = ArgumentParser()
+    parser.add_argument('--version', action='version', version=version_string)
     argparse.add_argument('--extract-fields', nargs='+', default=[],
                           help='Extract chronicles topics into CSV columns')
     argparse.add_argument('--constant-column', metavar='KEY=VALUE', nargs='+', type=kv_pair,

logtools/log/sources/input/elastic_search_source.py

@@ -11,6 +11,7 @@ from logtools.log.utils import tree
 logger = logging.getLogger(__name__)
 
 INDEX_PREFIX = 'continuous-tests-pods'
+ES_MAX_BATCH_SIZE = 10_000
 
 
 @dataclass
@@ -29,6 +30,8 @@ class ElasticSearchSource(LogSource[TimestampedLogLine[ElasticSearchLocation]]):
             client: Optional[Elasticsearch] = None,
             start_date: Optional[datetime] = None,
             end_date: Optional[datetime] = None,
+            limit: Optional[int] = None,
+            es_batch_size=ES_MAX_BATCH_SIZE
     ):
         if client is None:
             logger.warning('No client provided, defaulting to localhost')
@@ -40,6 +43,9 @@ class ElasticSearchSource(LogSource[TimestampedLogLine[ElasticSearchLocation]]):
         self.client = client
         self.start_date = start_date
         self.end_date = end_date
+        self.limit = limit
+        self.es_batch_size = es_batch_size
+        self.page_fetch_counter = 0
 
     def __iter__(self) -> Iterator[TimestampedLogLine[ElasticSearchLocation]]:
         for index in self._indices():
@@ -76,11 +82,11 @@ class ElasticSearchSource(LogSource[TimestampedLogLine[ElasticSearchLocation]]):
 
         if self.pods is not None:
             filters = query['query']['bool'].setdefault('filter', [])
-            filters.append({"terms": {"pod_name.keyword": list(self.pods)}})
+            filters.append({"terms": {"pod_name.keyword": list(self.pods)}})  # type: ignore
 
         if self.run_id is not None:
             filters = query['query']['bool'].setdefault('filter', [])
-            filters.append({"term": {"pod_labels.runid.keyword": self.run_id}})
+            filters.append({"term": {"pod_labels.runid.keyword": self.run_id}})  # type: ignore
 
         if 'query' not in query:
             query['query'] = {'match_all': {}}
@@ -88,8 +94,11 @@ class ElasticSearchSource(LogSource[TimestampedLogLine[ElasticSearchLocation]]):
         return query
 
     def _run_scan(self, query: Dict[str, Any], index: str):
-        # the search type stub does not contain the body argument for some reason so we disable typing here.
-        initial = self.client.search(index=index, body=query, size=5_000, scroll='2m')  # type: ignore
+        remaining = self.limit if self.limit is not None else float('inf')
+        # XXX the search type stub does not contain the body argument for some reason so we disable typing here.
+        initial = self.client.search(index=index, body=query, size=min(remaining, self.es_batch_size),
+                                     scroll='2m')  # type: ignore
+        self.page_fetch_counter += 1
         scroll_id = initial['_scroll_id']
         results = initial
 
@@ -97,12 +106,16 @@ class ElasticSearchSource(LogSource[TimestampedLogLine[ElasticSearchLocation]]):
             while True:
                 documents = results['hits']['hits']
                 if not documents:
-                    break
+                    return
 
                 for doc in documents:
                     yield doc
+                    remaining -= 1
+                    if remaining <= 0:
+                        return
 
                 results = self.client.scroll(scroll_id=scroll_id, scroll='2m')
+                self.page_fetch_counter += 1
         finally:
             self.client.clear_scroll(scroll_id=scroll_id)
 

logtools/log/sources/input/file_log_source.py

@@ -3,6 +3,7 @@ from pathlib import Path
 from typing import Iterator
 
 from logtools.log.base import LogSource, RawLogLine, LineNumberLocation
+from logtools.log.sources.input.textio_log_source import TextIOLogSource
 
 
 @dataclass
@@ -10,14 +11,13 @@ class FileLineLocation(LineNumberLocation):
     path: Path
 
 
-class FileLogSource(LogSource[RawLogLine[FileLineLocation]]):
+class FileLogSource(TextIOLogSource[FileLineLocation]):
     def __init__(self, path: Path):
+        super().__init__(path.open(encoding='utf-8'))
         self.path = path
 
-    def __iter__(self) -> Iterator[RawLogLine[FileLineLocation]]:
-        with self.path.open(encoding='utf-8') as infile:
-            for i, raw_string in enumerate(infile, start=1):
-                yield RawLogLine(
-                    location=FileLineLocation(path=self.path, line_number=i),
-                    raw=raw_string
-                )
+    def _location(self, line_no: int):
+        return FileLineLocation(path=self.path, line_number=line_no)
+
+    def _done(self):
+        self.source.close()

logtools/log/sources/input/string_log_source.py

@@ -3,6 +3,7 @@ from io import StringIO
 from typing import Iterator
 
 from logtools.log.base import LogSource, RawLogLine, LineNumberLocation
+from logtools.log.sources.input.textio_log_source import TextIOLogSource
 
 
 @dataclass
@@ -10,14 +11,10 @@ class ParseLocation(LineNumberLocation):
     name: str
 
 
-class StringLogSource(LogSource[RawLogLine[ParseLocation]]):
+class StringLogSource(TextIOLogSource[ParseLocation]):
     def __init__(self, lines: str, name: str = 'unnamed'):
+        super().__init__(StringIO(lines))
         self.name = name
-        self.stream = StringIO(lines)
 
-    def __iter__(self) -> Iterator[RawLogLine[ParseLocation]]:
-        for line_number, line in enumerate(self.stream, start=1):
-            yield RawLogLine(
-                location=ParseLocation(name=self.name, line_number=line_number),
-                raw=line
-            )
+    def _location(self, line_no: int):
+        return ParseLocation(name=self.name, line_number=line_no)

logtools/log/sources/input/tests/cassettes/test_elasticsearch_source/test_should_respect_fetching_limits_when_limit_larger_than_batch_size.yaml

@@ -0,0 +1,132 @@
+interactions:
+- request:
+    body: '{"query":{"bool":{"filter":[{"terms":{"pod_name.keyword":["codex1-3-6c565dbd66-wxg49"]}},{"term":{"pod_labels.runid.keyword":"20240208-115030"}}]}},"size":7,"sort":[{"@timestamp":"asc"}]}'
+    headers:
+      accept:
+      - application/vnd.elasticsearch+json; compatible-with=8
+      connection:
+      - keep-alive
+      content-type:
+      - application/vnd.elasticsearch+json; compatible-with=8
+      user-agent:
+      - elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0)
+      x-elastic-client-meta:
+      - es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7
+    method: POST
+    uri: http://localhost:9200/continuous-tests-pods-*/_search?scroll=2m
+  response:
+    body:
+      string: '{"_scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDnF1ZXJ5VGhlbkZldGNoDBZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YgWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YwWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YoWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YkWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YsWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y0WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y4WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZMWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZAWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y8WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZEWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZIWVmdpQjdNT1RUM3VuSmxXekR6ZnREZw==","took":13,"timed_out":false,"_shards":{"total":12,"successful":12,"skipped":0,"failed":0},"hits":{"total":{"value":80036,"relation":"eq"},"max_score":null,"hits":[{"_index":"continuous-tests-pods-2024.02.08","_id":"lpQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.918606306Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"Internal:
+        Set CODEX_NAT: 10.244.4.178","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563918]},{"_index":"continuous-tests-pods-2024.02.08","_id":"l5Qmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.918876891Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"Run
+        Codex node","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563918]},{"_index":"continuous-tests-pods-2024.02.08","_id":"mJQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.927052796Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"NOT
+        2024-02-08 23:56:03.926+00:00 Starting metrics HTTP server               tid=1
+        url=http://0.0.0.0:30006/metrics count=1","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563927]},{"_index":"continuous-tests-pods-2024.02.08","_id":"mZQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.934921297Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"WRN
+        2024-02-08 23:56:03.926+00:00 Data directory has insecure permissions. Correcting
+        them. tid=1 data_dir=datadir3 current_permissions=0755 required_permissions=0700
+        count=2","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563934]},{"_index":"continuous-tests-pods-2024.02.08","_id":"mpQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.934945015Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC
+        2024-02-08 23:56:03.926+00:00 Data dir initialized                       topics=\"codex\"
+        tid=1 dir=datadir3 count=3","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563934]},{"_index":"continuous-tests-pods-2024.02.08","_id":"m5Qmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.934955237Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC
+        2024-02-08 23:56:03.929+00:00 Repo dir initialized                       topics=\"codex\"
+        tid=1 dir=datadir3/repo count=4","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563934]},{"_index":"continuous-tests-pods-2024.02.08","_id":"nJQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.934960073Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"INF
+        2024-02-08 23:56:03.929+00:00 Creating a private key and saving it       tid=1
+        count=5","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563934]}]}}'
+    headers:
+      Transfer-Encoding:
+      - chunked
+      X-elastic-product:
+      - Elasticsearch
+      content-type:
+      - application/vnd.elasticsearch+json;compatible-with=8
+    status:
+      code: 200
+      message: OK
+- request:
+    body: '{"scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDnF1ZXJ5VGhlbkZldGNoDBZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YgWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YwWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YoWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YkWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YsWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y0WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y4WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZMWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZAWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y8WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZEWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZIWVmdpQjdNT1RUM3VuSmxXekR6ZnREZw==","scroll":"2m"}'
+    headers:
+      accept:
+      - application/vnd.elasticsearch+json; compatible-with=8
+      connection:
+      - keep-alive
+      content-type:
+      - application/vnd.elasticsearch+json; compatible-with=8
+      user-agent:
+      - elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0)
+      x-elastic-client-meta:
+      - es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7
+    method: POST
+    uri: http://localhost:9200/_search/scroll
+  response:
+    body:
+      string: '{"_scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDnF1ZXJ5VGhlbkZldGNoDBZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YgWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YwWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YoWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YkWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YsWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y0WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y4WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZMWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZAWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y8WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZEWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZIWVmdpQjdNT1RUM3VuSmxXekR6ZnREZw==","took":3,"timed_out":false,"_shards":{"total":12,"successful":12,"skipped":0,"failed":0},"hits":{"total":{"value":80036,"relation":"eq"},"max_score":null,"hits":[{"_index":"continuous-tests-pods-2024.02.08","_id":"nZQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.950492498Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC
+        2024-02-08 23:56:03.950+00:00 Updating announce record                   topics=\"codex
+        discovery\" tid=1 addrs=@[/ip4/0.0.0.0/tcp/8081] count=6","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563950]},{"_index":"continuous-tests-pods-2024.02.08","_id":"npQmi40BTe2pYqi1cfr7","_score":null,"_ignored":["message.keyword"],"_source":{"@timestamp":"2024-02-08T23:56:03.969109839Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC
+        2024-02-08 23:56:03.968+00:00 Starting codex node                        topics=\"codex
+        node\" tid=1 config=\"(configFile: none(InputFile), logLevel: \\\"TRACE;warn:discv5,providers,manager,cache;warn:libp2p,multistream,switch,transport,tcptransport,semaphore,asyncstreamwrapper,lpstream,mplex,mplexchannel,noise,bufferstream,mplexcoder,secure,chronosstream,connection,connmanager,websock,ws-session,dialer,muxedupgrade,upgrade,identify\\\",
+        logFormat: auto, metricsEnabled: true, metricsAddress: 0.0.0.0, metricsPort:
+        30006, dataDir: datadir3, profilerMaxMetrics: 100, cmd: noCommand, listenAddrs:
+        @[/ip4/0.0.0.0/tcp/8081], nat: 10.244.4.178, discoveryIp: 0.0.0.0, discoveryPort:
+        8080, netPrivKeyFile: \\\"key\\\", bootstrapNodes: @[(envelope: (publicKey:
+        secp256k1 key (040b1beda7b6b56401c2c29554df74729e75b0906d0e8848dcc5dd04b28d2fcad4bd0b489cdc7a5a3a1c0e9ec0d0d3d8004c946da4baa1752fcb201feda1304d3b),
+        domain: \\\"libp2p-peer-record\\\", payloadType: @[3, 1], payload: @[10, 39,
+        0, 37, 8, 2, 18, 33, 3, 11, 27, 237, 167, 182, 181, 100, 1, 194, 194, 149,
+        84, 223, 116, 114, 158, 117, 176, 144, 109, 14, 136, 72, 220, 197, 221, 4,
+        178, 141, 47, 202, 212, 16, 178, 210, 149, 174, 6, 26, 11, 10, 9, 4, 10, 244,
+        2, 44, 145, 2, 31, 144], signature: 304402201C49248DA73B2055951C115F46301D8D2B72FAF83423CC45F69E9BED8F1E9679022076A752567FD7D7E7A262D90121C9B69DA9D76648728E23C1DAE6EDBB6B2A6811),
+        data: (peerId: 16Uiu2HAmDQMXzu725FEnd6pgzJr6iLJGt8TffyNK673VBArAMptT, seqNo:
+        1707436338, addresses: @[(address: /ip4/10.244.2.44/udp/8080)]))], maxPeers:
+        160, agentString: \\\"Codex\\\", apiBindAddress: \\\"0.0.0.0\\\", apiPort:
+        30005, repoKind: fs, storageQuota: 62914560000\\''NByte, blockTtl: 16w3d17h46m39s,
+        blockMaintenanceInterval: 16w3d17h46m39s, blockMaintenanceNumberOfBlocks:
+        10000, cacheSize: 0\\''NByte, persistence: false, ethProvider: \\\"ws://localhost:8545\\\",
+        ethAccount: none(Address), ethPrivateKey: none(string), marketplaceAddress:
+        none(Address), validator: false, validatorMaxSlots: 1000, simulateProofFailures:
+        0, logFile: none(string))\" count=7","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563969]},{"_index":"continuous-tests-pods-2024.02.08","_id":"n5Qmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.969228477Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC
+        2024-02-08 23:56:03.968+00:00 Starting repo                              topics=\"codex
+        repostore\" tid=1 count=8","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563969]},{"_index":"continuous-tests-pods-2024.02.08","_id":"oJQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.969273960Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC
+        2024-02-08 23:56:03.969+00:00 Number of blocks in store at start         topics=\"codex
+        repostore\" tid=1 total=0 count=9","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563969]},{"_index":"continuous-tests-pods-2024.02.08","_id":"oZQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.969452435Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"NOT
+        2024-02-08 23:56:03.969+00:00 Current bytes used for cache quota         topics=\"codex
+        repostore\" tid=1 bytes=0 count=10","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563969]},{"_index":"continuous-tests-pods-2024.02.08","_id":"opQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.969572555Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"NOT
+        2024-02-08 23:56:03.969+00:00 Current bytes used for persist quota       topics=\"codex
+        repostore\" tid=1 bytes=0 count=11","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563969]},{"_index":"continuous-tests-pods-2024.02.08","_id":"o5Qmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.969654771Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC
+        2024-02-08 23:56:03.969+00:00 Timer starting:                            tid=1
+        name=\"Unnamed Timer\" count=12","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563969]}]}}'
+    headers:
+      Transfer-Encoding:
+      - chunked
+      X-elastic-product:
+      - Elasticsearch
+      content-type:
+      - application/vnd.elasticsearch+json;compatible-with=8
+    status:
+      code: 200
+      message: OK
+- request:
+    body: '{"scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDnF1ZXJ5VGhlbkZldGNoDBZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YgWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YwWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YoWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YkWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YsWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y0WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y4WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZMWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZAWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7Y8WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZEWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7ZIWVmdpQjdNT1RUM3VuSmxXekR6ZnREZw=="}'
+    headers:
+      accept:
+      - application/vnd.elasticsearch+json; compatible-with=8
+      connection:
+      - keep-alive
+      content-type:
+      - application/vnd.elasticsearch+json; compatible-with=8
+      user-agent:
+      - elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0)
+      x-elastic-client-meta:
+      - es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7
+    method: DELETE
+    uri: http://localhost:9200/_search/scroll
+  response:
+    body:
+      string: '{"succeeded":true,"num_freed":12}'
+    headers:
+      X-elastic-product:
+      - Elasticsearch
+      content-length:
+      - '33'
+      content-type:
+      - application/vnd.elasticsearch+json;compatible-with=8
+    status:
+      code: 200
+      message: OK
+version: 1

logtools/log/sources/input/tests/cassettes/test_elasticsearch_source/test_should_respect_fetching_limits_when_limit_smaller_than_batch_size.yaml

@@ -0,0 +1,96 @@
+interactions:
+- request:
+    body: '{"query":{"bool":{"filter":[{"terms":{"pod_name.keyword":["codex1-3-6c565dbd66-wxg49"]}},{"term":{"pod_labels.runid.keyword":"20240208-115030"}}]}},"size":10,"sort":[{"@timestamp":"asc"}]}'
+    headers:
+      accept:
+      - application/vnd.elasticsearch+json; compatible-with=8
+      connection:
+      - keep-alive
+      content-type:
+      - application/vnd.elasticsearch+json; compatible-with=8
+      user-agent:
+      - elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0)
+      x-elastic-client-meta:
+      - es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7
+    method: POST
+    uri: http://localhost:9200/continuous-tests-pods-*/_search?scroll=2m
+  response:
+    body:
+      string: '{"_scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDnF1ZXJ5VGhlbkZldGNoDBZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7XwWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7X0WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7X4WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7X8WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YAWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YEWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YIWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YMWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YQWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YUWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YYWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YcWVmdpQjdNT1RUM3VuSmxXekR6ZnREZw==","took":10,"timed_out":false,"_shards":{"total":12,"successful":12,"skipped":0,"failed":0},"hits":{"total":{"value":80036,"relation":"eq"},"max_score":null,"hits":[{"_index":"continuous-tests-pods-2024.02.08","_id":"lpQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.918606306Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"Internal:
+        Set CODEX_NAT: 10.244.4.178","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563918]},{"_index":"continuous-tests-pods-2024.02.08","_id":"l5Qmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.918876891Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"Run
+        Codex node","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563918]},{"_index":"continuous-tests-pods-2024.02.08","_id":"mJQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.927052796Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"NOT
+        2024-02-08 23:56:03.926+00:00 Starting metrics HTTP server               tid=1
+        url=http://0.0.0.0:30006/metrics count=1","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563927]},{"_index":"continuous-tests-pods-2024.02.08","_id":"mZQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.934921297Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"WRN
+        2024-02-08 23:56:03.926+00:00 Data directory has insecure permissions. Correcting
+        them. tid=1 data_dir=datadir3 current_permissions=0755 required_permissions=0700
+        count=2","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563934]},{"_index":"continuous-tests-pods-2024.02.08","_id":"mpQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.934945015Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC
+        2024-02-08 23:56:03.926+00:00 Data dir initialized                       topics=\"codex\"
+        tid=1 dir=datadir3 count=3","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563934]},{"_index":"continuous-tests-pods-2024.02.08","_id":"m5Qmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.934955237Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC
+        2024-02-08 23:56:03.929+00:00 Repo dir initialized                       topics=\"codex\"
+        tid=1 dir=datadir3/repo count=4","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563934]},{"_index":"continuous-tests-pods-2024.02.08","_id":"nJQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.934960073Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"INF
+        2024-02-08 23:56:03.929+00:00 Creating a private key and saving it       tid=1
+        count=5","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563934]},{"_index":"continuous-tests-pods-2024.02.08","_id":"nZQmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.950492498Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC
+        2024-02-08 23:56:03.950+00:00 Updating announce record                   topics=\"codex
+        discovery\" tid=1 addrs=@[/ip4/0.0.0.0/tcp/8081] count=6","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563950]},{"_index":"continuous-tests-pods-2024.02.08","_id":"npQmi40BTe2pYqi1cfr7","_score":null,"_ignored":["message.keyword"],"_source":{"@timestamp":"2024-02-08T23:56:03.969109839Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC
+        2024-02-08 23:56:03.968+00:00 Starting codex node                        topics=\"codex
+        node\" tid=1 config=\"(configFile: none(InputFile), logLevel: \\\"TRACE;warn:discv5,providers,manager,cache;warn:libp2p,multistream,switch,transport,tcptransport,semaphore,asyncstreamwrapper,lpstream,mplex,mplexchannel,noise,bufferstream,mplexcoder,secure,chronosstream,connection,connmanager,websock,ws-session,dialer,muxedupgrade,upgrade,identify\\\",
+        logFormat: auto, metricsEnabled: true, metricsAddress: 0.0.0.0, metricsPort:
+        30006, dataDir: datadir3, profilerMaxMetrics: 100, cmd: noCommand, listenAddrs:
+        @[/ip4/0.0.0.0/tcp/8081], nat: 10.244.4.178, discoveryIp: 0.0.0.0, discoveryPort:
+        8080, netPrivKeyFile: \\\"key\\\", bootstrapNodes: @[(envelope: (publicKey:
+        secp256k1 key (040b1beda7b6b56401c2c29554df74729e75b0906d0e8848dcc5dd04b28d2fcad4bd0b489cdc7a5a3a1c0e9ec0d0d3d8004c946da4baa1752fcb201feda1304d3b),
+        domain: \\\"libp2p-peer-record\\\", payloadType: @[3, 1], payload: @[10, 39,
+        0, 37, 8, 2, 18, 33, 3, 11, 27, 237, 167, 182, 181, 100, 1, 194, 194, 149,
+        84, 223, 116, 114, 158, 117, 176, 144, 109, 14, 136, 72, 220, 197, 221, 4,
+        178, 141, 47, 202, 212, 16, 178, 210, 149, 174, 6, 26, 11, 10, 9, 4, 10, 244,
+        2, 44, 145, 2, 31, 144], signature: 304402201C49248DA73B2055951C115F46301D8D2B72FAF83423CC45F69E9BED8F1E9679022076A752567FD7D7E7A262D90121C9B69DA9D76648728E23C1DAE6EDBB6B2A6811),
+        data: (peerId: 16Uiu2HAmDQMXzu725FEnd6pgzJr6iLJGt8TffyNK673VBArAMptT, seqNo:
+        1707436338, addresses: @[(address: /ip4/10.244.2.44/udp/8080)]))], maxPeers:
+        160, agentString: \\\"Codex\\\", apiBindAddress: \\\"0.0.0.0\\\", apiPort:
+        30005, repoKind: fs, storageQuota: 62914560000\\''NByte, blockTtl: 16w3d17h46m39s,
+        blockMaintenanceInterval: 16w3d17h46m39s, blockMaintenanceNumberOfBlocks:
+        10000, cacheSize: 0\\''NByte, persistence: false, ethProvider: \\\"ws://localhost:8545\\\",
+        ethAccount: none(Address), ethPrivateKey: none(string), marketplaceAddress:
+        none(Address), validator: false, validatorMaxSlots: 1000, simulateProofFailures:
+        0, logFile: none(string))\" count=7","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563969]},{"_index":"continuous-tests-pods-2024.02.08","_id":"n5Qmi40BTe2pYqi1cfr7","_score":null,"_source":{"@timestamp":"2024-02-08T23:56:03.969228477Z","container_image":"docker.io/codexstorage/nim-codex:sha-a3c95be-dist-tests","container_name":"codex1-3","file":"/var/log/pods/two-client-test-bugfix_codex1-3-6c565dbd66-wxg49_0ad5dc26-09e2-47e4-9257-c7dc372068d4/codex1-3/0.log","message":"TRC
+        2024-02-08 23:56:03.968+00:00 Starting repo                              topics=\"codex
+        repostore\" tid=1 count=8","pod_labels":{"app":"codex","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexdiscordbotid":"thatbenbierens-codex-discordbot-initial","codexid":"docker.io-codexstorage/nim-codex:sha-a3c95be-dist-tests","codexrevision":"unknown","gethid":"codexstorage/dist-tests-geth:latest","pod-template-hash":"6c565dbd66","pod-uuid":"253f4886-a27b-4398-8d46-9f86fd1ded9c","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","testid":"envvar-testid-notset","tests-type":"continuous-tests"},"pod_name":"codex1-3-6c565dbd66-wxg49","pod_namespace":"two-client-test-bugfix","pod_node_name":"tests-s-4vcpu-8gb-ojxa3"},"sort":[1707436563969]}]}}'
+    headers:
+      Transfer-Encoding:
+      - chunked
+      X-elastic-product:
+      - Elasticsearch
+      content-type:
+      - application/vnd.elasticsearch+json;compatible-with=8
+    status:
+      code: 200
+      message: OK
+- request:
+    body: '{"scroll_id":"FGluY2x1ZGVfY29udGV4dF91dWlkDnF1ZXJ5VGhlbkZldGNoDBZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7XwWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7X0WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7X4WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7X8WVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YAWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YEWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YIWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YMWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YQWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YUWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YYWVmdpQjdNT1RUM3VuSmxXekR6ZnREZxZ4RF9qaURMaFJWT19DcWhQTTVxdzZnAAAAAAAl7YcWVmdpQjdNT1RUM3VuSmxXekR6ZnREZw=="}'
+    headers:
+      accept:
+      - application/vnd.elasticsearch+json; compatible-with=8
+      connection:
+      - keep-alive
+      content-type:
+      - application/vnd.elasticsearch+json; compatible-with=8
+      user-agent:
+      - elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0)
+      x-elastic-client-meta:
+      - es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7
+    method: DELETE
+    uri: http://localhost:9200/_search/scroll
+  response:
+    body:
+      string: '{"succeeded":true,"num_freed":12}'
+    headers:
+      X-elastic-product:
+      - Elasticsearch
+      content-length:
+      - '33'
+      content-type:
+      - application/vnd.elasticsearch+json;compatible-with=8
+    status:
+      code: 200
+      message: OK
+version: 1

logtools/log/sources/input/tests/test_elasticsearch_source.py

@@ -3,7 +3,7 @@ from datetime import timedelta
 import pytest
 from dateutil import parser
 
-from logtools.log.sources.input.elastic_search.elastic_search_source import ElasticSearchSource
+from logtools.log.sources.input.elastic_search_source import ElasticSearchSource
 
 
 @pytest.mark.vcr
@@ -28,6 +28,7 @@ def test_should_fetch_logs_by_date():
     assert {line.location.pod_name for line in lines} == {'codex1-3-b558568cf-tvcsc', 'bootstrap-2-58b69484bc-88msf'}
     assert {line.location.run_id for line in lines} == {'20231109-101554'}
 
+
 @pytest.mark.vcr
 def test_should_fetch_logs_when_no_dates_are_specified():
     log = ElasticSearchSource(
@@ -39,3 +40,30 @@ def test_should_fetch_logs_when_no_dates_are_specified():
         next(log.__iter__())
     except StopIteration:
         assert False, "Should have returned at least one log line"
+
+
+@pytest.mark.vcr
+def test_should_respect_fetching_limits_when_limit_smaller_than_batch_size():
+    log = ElasticSearchSource(
+        run_id='20240208-115030',
+        pods={'codex1-3-6c565dbd66-wxg49'},
+        limit=10
+    )
+
+    lines = list(log)
+    assert len(lines) == 10
+    assert log.page_fetch_counter == 1
+
+
+@pytest.mark.vcr
+def test_should_respect_fetching_limits_when_limit_larger_than_batch_size():
+    log = ElasticSearchSource(
+        run_id='20240208-115030',
+        pods={'codex1-3-6c565dbd66-wxg49'},
+        es_batch_size=7,
+        limit=10
+    )
+
+    lines = list(log)
+    assert len(lines) == 10
+    assert log.page_fetch_counter == 2

logtools/log/sources/input/textio_log_source.py

@@ -0,0 +1,26 @@
+from typing import TypeVar, TextIO, Iterator
+
+from logtools.log.base import LineNumberLocation, RawLogLine, LogSource
+
+TTextIOLineLocation = TypeVar('TTextIOLineLocation', bound=LineNumberLocation)
+
+
+class TextIOLogSource(LogSource[RawLogLine[TTextIOLineLocation]]):
+    def __init__(self, source: TextIO):
+        self.source = source
+
+    def __iter__(self) -> Iterator[RawLogLine[TTextIOLineLocation]]:
+        try:
+            for i, raw_string in enumerate(self.source, start=1):
+                yield RawLogLine(
+                    location=self._location(i),
+                    raw=raw_string
+                )
+        finally:
+            self._done()
+
+    def _location(self, line_no: int):
+        return LineNumberLocation(line_number=line_no)
+
+    def _done(self):
+        pass

logtools/log/sources/parse/chronicles_raw_source.py

@@ -10,13 +10,13 @@ from dateutil import parser
 from logtools.log.base import LogSource, TLocation, RawLogLine, TimestampedLogLine
 
 _LOG_LINE = re.compile(
-    r'(?P<line_type>\w{3}) (?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}\+\d{2}:\d{2}) (?P<message>.*) '
+    r'(?P<line_type>\w{3}) (?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}[+-]\d{2}:\d{2}) (?P<message>.*) '
     r'count=(?P<count>\d+)$'
 )
 
-_TOPICS = re.compile(r'((\w+=("[\w\s]+"|\S+) )+)?\w+=("[\w\s]+"|\S+)$')
+_TOPICS = re.compile(r'((\w+=("[^"]+"|\S+) )+)?\w+=("(\\"|[^"])+"|\S+)$')
 
-_TOPICS_KV = re.compile(r'(?P<key>\w+)=(?P<value>"[\w\s]+"|\S+)')
+_TOPICS_KV = re.compile(r'(?P<key>\w+)=(?P<value>"(?:\\"|[^"])+"|\S+)')
 
 
 class LogLevel(Enum):

logtools/log/sources/parse/tests/test_chronicles_raw_source.py

@@ -7,15 +7,15 @@ from logtools.log.sources.input.string_log_source import StringLogSource
 from logtools.log.sources.parse.chronicles_raw_source import ChroniclesRawSource, ChroniclesLogLine, LogLevel
 
 
-def test_should_parse_raw_chronicles_logs():
-    source = ChroniclesRawSource(
-        StringLogSource(
-            lines='TRC 2023-10-16 17:28:46.579+00:00 Sending want list to peer                  '
-                  'topics="codex blockexcnetwork" tid=1 peer=16U*7mogoM type=WantBlock items=1 count=870781'
-        )
-    ).__iter__()
+def parse_single_line(lines: str):
+    return next(ChroniclesRawSource(StringLogSource(lines)).__iter__())
 
-    line = next(source)
+
+def test_should_parse_raw_chronicles_logs():
+    line = parse_single_line(
+        'TRC 2023-10-16 17:28:46.579+00:00 Sending want list to peer                  '
+        'topics="codex blockexcnetwork" tid=1 peer=16U*7mogoM type=WantBlock items=1 count=870781'
+    )
 
     assert line.level == LogLevel.trace
     assert line.timestamp == datetime(2023, 10, 16, 17, 28, 46,
@@ -50,3 +50,57 @@ def test_should_parse_chronicles_fields():
         'type': 'WantBlock',
         'items': '1',
     }
+
+
+def test_should_parse_topics_with_non_alphanumeric_character_values():
+    line = parse_single_line(
+        'WRN 2024-02-02 20:38:47.316+00:00 a message topics="codex pendingblocks" address="cid: zDx*QP4zx9" '
+        'count=10641'
+    )
+
+    assert line.message == "a message"
+    assert line.count == 10641
+    assert line.fields == {
+        'topics': '"codex pendingblocks"',
+        'address': '"cid: zDx*QP4zx9"',
+    }
+
+
+def test_should_parse_topics_with_escaped_quotes_in_values():
+    line = parse_single_line(
+        'WRN 2024-02-02 20:38:47.316+00:00 a message topics="codex pendingblocks" '
+        'address="cid: \\"zDx*QP4zx9\\"" count=10641'
+    )
+
+    assert line.message == "a message"
+    assert line.count == 10641
+    assert line.fields == {
+        'topics': '"codex pendingblocks"',
+        'address': '"cid: \\"zDx*QP4zx9\\""',
+    }
+
+
+def test_should_parse_topics_with_escaped_quotes_and_backlashes_in_value():
+    line = parse_single_line(
+        'TRC 2024-02-02 20:37:18.316+00:00 Starting codex node     topics="codex node" '
+        'config="some \\"quoted\\" string with \'more\' escape chars" count=7'
+    )
+
+    assert line.message == "Starting codex node"
+    assert line.count == 7
+    assert line.fields == {
+        'topics': '"codex node"',
+        'config': '"some \\"quoted\\" string with \'more\' escape chars"',
+    }
+
+
+def test_should_parse_timestamps_with_negative_time_offsets():
+    line = parse_single_line(
+        'WRN 2024-02-02 20:38:47.316-03:00 a message topics="codex pendingblocks" address="cid: zDx*QP4zx9" '
+        'count=10641'
+    )
+
+    assert line.timestamp == datetime(2024, 2, 2, 20, 38, 47, 316000,
+                                      tzinfo=pytz.FixedOffset(-180))
+    assert line.message == "a message"
+    assert line.count == 10641

logtools/log/utils.py

@@ -1,8 +1,8 @@
 from collections import defaultdict
 
-from typing import Dict
+from typing import Dict, Any
 
-Tree = Dict[str, 'Tree']
+Tree = Dict[str, 'Tree'] | Any
 
 
 def tree() -> Tree:

logtools/resource/core.py

@@ -0,0 +1,62 @@
+import abc
+from dataclasses import dataclass
+from datetime import datetime
+from enum import Enum
+from typing import Iterator, Optional, List
+
+
+@dataclass(frozen=True)
+class Namespace:
+    name: str
+    run_id: tuple[str, ...]
+    indices: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class Pod:
+    name: str
+    namespace: str
+    run_id: str
+    indices: tuple[str, ...]
+
+
+class TestStatus(Enum):
+    passed = 'passed'
+    failed = 'failed'
+
+
+@dataclass(frozen=True)
+class SummarizedTestRun:
+    id: str
+    run_id: str
+    test_name: str
+    pods: List[str]
+    start: datetime
+    end: datetime
+    duration: float
+    status: TestStatus
+
+
+@dataclass(frozen=True)
+class TestRun:
+    test_run: SummarizedTestRun
+    error: Optional[str]
+    stacktrace: Optional[str]
+
+
+class Repository(abc.ABC):
+    @abc.abstractmethod
+    def namespaces(self, prefix: Optional[str] = None) -> Iterator[Namespace]:
+        ...
+
+    @abc.abstractmethod
+    def pods(self, prefix: Optional[str] = None, run_id: Optional[str] = None):
+        ...
+
+    @abc.abstractmethod
+    def test_runs(self, run_id: str, failed_only=False) -> Iterator[SummarizedTestRun]:
+        ...
+
+    @abc.abstractmethod
+    def test_run(self, test_run_id: str) -> TestRun:
+        ...

logtools/resource/elastic_search_log_repo.py

@@ -1,36 +1,25 @@
 import logging
-from dataclasses import dataclass
-from datetime import datetime
+from datetime import datetime, timedelta
 from typing import Optional, Iterator, Dict, Any
 
+from dateutil import parser
 from elasticsearch import Elasticsearch
 
+from logtools.resource.core import Repository, TestRun, TestStatus, SummarizedTestRun, Namespace, Pod
+
 logger = logging.getLogger(__name__)
 
-
-@dataclass(frozen=True)
-class Namespace:
-    name: str
-    run_id: tuple[str, ...]
-    indices: tuple[str, ...]
-
-
-@dataclass(frozen=True)
-class Pod:
-    name: str
-    namespace: str
-    run_id: str
-    indices: tuple[str, ...]
-
-
 MAX_AGGREGATION_BUCKETS = 1000
 
+POD_LOGS_INDEX_SET = 'continuous-tests-pods-*'
+TEST_STATUS_INDEX_SET = 'continuous-tests-status-*'
+
+
+class ElasticSearchLogRepo(Repository):
 
-class ElasticSearchLogRepo:
     def __init__(
             self,
             client: Optional[Elasticsearch] = None,
-            indices: str = 'continuous-tests-pods-*',
             since: Optional[datetime] = None,
     ):
         if client is None:
@@ -38,7 +27,6 @@ class ElasticSearchLogRepo:
             client = Elasticsearch(hosts='http://localhost:9200', request_timeout=60)
 
         self.client = client
-        self.indices = indices
         self.since = since
 
     def namespaces(self, prefix: Optional[str] = None) -> Iterator[Namespace]:
@@ -58,7 +46,7 @@ class ElasticSearchLogRepo:
         if prefix is not None:
             query['aggs']['distinct_namespaces']['terms']['include'] = f'{prefix}.*'  # type: ignore
 
-        result = self.client.search(index=self.indices, body=query)  # type: ignore
+        result = self.client.search(index=POD_LOGS_INDEX_SET, body=query)  # type: ignore
 
         for namespace in result['aggregations']['distinct_namespaces']['buckets']:
             yield Namespace(
@@ -92,7 +80,7 @@ class ElasticSearchLogRepo:
                 }
             }
 
-        for pod in self.client.search(index=self.indices,
+        for pod in self.client.search(index=POD_LOGS_INDEX_SET,
                                       body=query)['aggregations']['distinct_pods']['buckets']:  # type: ignore
             assert len(pod['namespace']['buckets']) == 1, 'Pods should only have one namespace'
             assert len(pod['runid']['buckets']) == 1, 'Pods should only have one run_id'
@@ -104,6 +92,48 @@ class ElasticSearchLogRepo:
                 indices=tuple(sorted(index['key'] for index in pod['indices']['buckets']))
             )
 
+    def test_runs(self, run_id: str, failed_only=False) -> Iterator[SummarizedTestRun]:
+        query = self._time_limited({
+            'query': {
+                'bool': {
+                    'filter': [{'term': {'runid.keyword': run_id}}]
+                }
+            },
+            'sort': [{'@timestamp': 'desc'}],
+            'size': 10000,
+        })
+
+        if failed_only:
+            query['query']['bool']['filter'].append({'term': {'status.keyword': 'Failed'}})
+
+        for document in self.client.search(index=TEST_STATUS_INDEX_SET, body=query)['hits']['hits']:  # type: ignore
+            yield self._test_run_from_document(document['_index'], document['_id'], document['_source'])
+
+    def test_run(self, test_run_id: str) -> TestRun:
+        index, doc_id = test_run_id.split('/')
+        document = self.client.get(index=index, id=doc_id)
+        source = document['_source']
+        return TestRun(
+            test_run=self._test_run_from_document(document['_index'], document['_id'], source),
+            error=source.get('error'),
+            stacktrace=source.get('message'),
+        )
+
+    @staticmethod
+    def _test_run_from_document(index: str, doc_id: str, source: Dict[str, str]) -> SummarizedTestRun:
+        start = parser.parse(source['teststart'])
+        duration = float(source['testduration'])
+        return SummarizedTestRun(
+            id=f"{index}/{doc_id}",
+            run_id=source['runid'],
+            test_name=source['testname'],
+            start=start,
+            end=start + timedelta(seconds=duration),
+            duration=duration,
+            status=TestStatus(source['status'].lower()),
+            pods=source['involvedpodnames'].split(','),
+        )
+
     def _time_limited(self, query: Dict[str, Any]) -> Dict[str, Any]:
         if self.since is not None:
             query['query'] = {

logtools/resource/tests/cassettes/test_elasticsearch_log_repo/test_should_retrieve_failing_test_runs_over_a_single_run_id.yaml

@@ -0,0 +1,294 @@
+interactions:
+- request:
+    body: '{"query":{"bool":{"filter":[{"term":{"runid.keyword":"20240208-115030"}},{"term":{"status.keyword":"Failed"}}]}},"size":10000,"sort":[{"@timestamp":"desc"}]}'
+    headers:
+      accept:
+      - application/vnd.elasticsearch+json; compatible-with=8
+      connection:
+      - keep-alive
+      content-type:
+      - application/vnd.elasticsearch+json; compatible-with=8
+      user-agent:
+      - elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0)
+      x-elastic-client-meta:
+      - es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7
+    method: POST
+    uri: http://localhost:9200/continuous-tests-status-*/_search
+  response:
+    body:
+      string: '{"took":5,"timed_out":false,"_shards":{"total":25,"successful":25,"skipped":0,"failed":0},"hits":{"total":{"value":2,"relation":"eq"},"max_score":null,"hits":[{"_index":"continuous-tests-status-2024.02.09","_id":"VEbGi40BTe2pYqi1mE3V","_score":null,"_ignored":["message.keyword"],"_source":{"@timestamp":"2024-02-09T02:50:57.1125777Z","avgupload":"1.2MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":" data/zDvZRwzkz4EQMAECwa519qerKWmxMfeEgy4hneFg3Yr1xZZ7jHBU/network''
+        timed out after 3 tries over 10 mins, 1 secs.","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"bootstrap-2-75ffd985d5-sqhj5,codex2-4-8574cf7cdf-z75qr","involvedpods":"<BOOTSTRAP2>,<CODEX24>","message":"System.Exception:
+        System.TimeoutException: Retry ''HTTP-GET-STREAM: data/zDvZRwzkz4EQMAECwa519qerKWmxMfeEgy4hneFg3Yr1xZZ7jHBU/network''
+        timed out after 3 tries over 10 mins, 1 secs.\n ---> System.AggregateException:
+        One or more errors occurred. (One or more errors occurred. (A task was canceled.))
+        (One or more errors occurred. (A task was canceled.)) (One or more errors
+        occurred. (A task was canceled.)) (One or more errors occurred. (Connection
+        refused (codex2-4-8-int.two-client-test-bugfix.svc.cluster.local:30007)))\n
+        ---> System.AggregateException: One or more errors occurred. (A task was canceled.)\n
+        ---> System.Threading.Tasks.TaskCanceledException: A task was canceled.\n   at
+        System.Threading.Tasks.Task.GetExceptions(Boolean includeTaskCanceledExceptions)\n   at
+        System.Threading.Tasks.Task.ThrowIfExceptional(Boolean includeTaskCanceledExceptions)\n   at
+        System.Threading.Tasks.Task.Wait(Int32 millisecondsTimeout, CancellationToken
+        cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at Utils.Time.Wait[T](Task`1
+        task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0()
+        in /cs-codex-dist-tests/Framework/Core/Http.cs:line 125\n   at Utils.Time.Retry[T](Func`1
+        action, Int32 maxRetries, TimeSpan retryTime, String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
+        144\n   at Core.Http.LockRetry[T](Func`1 operation, String description) in
+        /cs-codex-dist-tests/Framework/Core/Http.cs:line 212\n   at Core.Http.HttpGetStream(String
+        route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line 120\n   at CodexPlugin.CodexAccess.DownloadFile(String
+        contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line
+        71\n   at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile
+        file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
+        200\n   at CodexPlugin.CodexNode.<>c__DisplayClass35_0.<DownloadContent>b__0()
+        in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n   at
+        Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug)
+        in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n   at CodexPlugin.CodexNode.DownloadContent(ContentId
+        contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
+        128\n   at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.<DownloadTestFile>b__0()
+        in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        43\n   at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action
+        action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        82\n   at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        43\n   at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n   at
+        System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
+        invokeAttr)\n   at System.Reflection.RuntimeMethodInfo.Invoke(Object obj,
+        BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n   at
+        ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke)
+        in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n   at
+        ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        251\n   at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        150\n   at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
+        /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n   at
+        ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.<Run>b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        53\n   at Utils.TaskFactory.CatchException(Action action, String name) in
+        /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n   at Utils.TaskFactory.<>c__DisplayClass2_0.<Run>b__0()
+        in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n   at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread
+        threadPoolThread, ExecutionContext executionContext, ContextCallback callback,
+        Object state)\n   at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task&
+        currentTaskSlot, Thread threadPoolThread)\n   at System.Threading.ThreadPoolWorkQueue.Dispatch()\n   at
+        System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n---
+        End of stack trace from previous location ---\n\n   --- End of inner exception
+        stack trace ---\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
+        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
+        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
+        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
+        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
+        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
+        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n   ---
+        End of inner exception stack trace ---\n ---> (Inner Exception #1) System.AggregateException:
+        One or more errors occurred. (A task was canceled.)\n ---> System.Threading.Tasks.TaskCanceledException:
+        A task was canceled.\n   at System.Threading.Tasks.Task.GetExceptions(Boolean
+        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
+        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
+        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
+        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
+        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
+        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
+        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n   at
+        Core.Http.LockRetry[T](Func`1 operation, String description) in /cs-codex-dist-tests/Framework/Core/Http.cs:line
+        212\n   at Core.Http.HttpGetStream(String route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line
+        120\n   at CodexPlugin.CodexAccess.DownloadFile(String contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line
+        71\n   at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile
+        file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
+        200\n   at CodexPlugin.CodexNode.<>c__DisplayClass35_0.<DownloadContent>b__0()
+        in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n   at
+        Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug)
+        in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n   at CodexPlugin.CodexNode.DownloadContent(ContentId
+        contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
+        128\n   at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.<DownloadTestFile>b__0()
+        in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        43\n   at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action
+        action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        82\n   at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        43\n   at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n   at
+        System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
+        invokeAttr)\n   at System.Reflection.RuntimeMethodInfo.Invoke(Object obj,
+        BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n   at
+        ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke)
+        in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n   at
+        ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        251\n   at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        150\n   at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
+        /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n   at
+        ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.<Run>b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        53\n   at Utils.TaskFactory.CatchException(Action action, String name) in
+        /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n   at Utils.TaskFactory.<>c__DisplayClass2_0.<Run>b__0()
+        in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n   at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread
+        threadPoolThread, ExecutionContext executionContext, ContextCallback callback,
+        Object state)\n   at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task&
+        currentTaskSlot, Thread threadPoolThread)\n   at System.Threading.ThreadPoolWorkQueue.Dispatch()\n   at
+        System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n---
+        End of stack trace from previous location ---\n\n   --- End of inner exception
+        stack trace ---\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
+        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
+        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
+        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
+        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
+        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
+        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n
+        ---> (Inner Exception #2) System.AggregateException: One or more errors occurred.
+        (A task was canceled.)\n ---> System.Threading.Tasks.TaskCanceledException:
+        A task was canceled.\n   at System.Threading.Tasks.Task.GetExceptions(Boolean
+        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
+        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
+        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
+        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
+        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
+        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
+        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n   at
+        Core.Http.LockRetry[T](Func`1 operation, String description) in /cs-codex-dist-tests/Framework/Core/Http.cs:line
+        212\n   at Core.Http.HttpGetStream(String route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line
+        120\n   at CodexPlugin.CodexAccess.DownloadFile(String contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line
+        71\n   at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile
+        file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
+        200\n   at CodexPlugin.CodexNode.<>c__DisplayClass35_0.<DownloadContent>b__0()
+        in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n   at
+        Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug)
+        in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n   at CodexPlugin.CodexNode.DownloadContent(ContentId
+        contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
+        128\n   at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.<DownloadTestFile>b__0()
+        in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        43\n   at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action
+        action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        82\n   at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        43\n   at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n   at
+        System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
+        invokeAttr)\n   at System.Reflection.RuntimeMethodInfo.Invoke(Object obj,
+        BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n   at
+        ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke)
+        in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n   at
+        ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        251\n   at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        150\n   at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
+        /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n   at
+        ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.<Run>b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        53\n   at Utils.TaskFactory.CatchException(Action action, String name) in
+        /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n   at Utils.TaskFactory.<>c__DisplayClass2_0.<Run>b__0()
+        in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n   at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread
+        threadPoolThread, ExecutionContext executionContext, ContextCallback callback,
+        Object state)\n   at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task&
+        currentTaskSlot, Thread threadPoolThread)\n   at System.Threading.ThreadPoolWorkQueue.Dispatch()\n   at
+        System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n---
+        End of stack trace from previous location ---\n\n   --- End of inner exception
+        stack trace ---\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
+        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
+        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
+        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
+        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
+        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
+        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n
+        ---> (Inner Exception #3) System.AggregateException: One or more errors occurred.
+        (Connection refused (codex2-4-8-int.two-client-test-bugfix.svc.cluster.local:30007))\n
+        ---> System.Net.Http.HttpRequestException: Connection refused (codex2-4-8-int.two-client-test-bugfix.svc.cluster.local:30007)\n
+        ---> System.Net.Sockets.SocketException (111): Connection refused\n   at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.ThrowException(SocketError
+        error, CancellationToken cancellationToken)\n   at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.System.Threading.Tasks.Sources.IValueTaskSource.GetResult(Int16
+        token)\n   at System.Net.Sockets.Socket.<ConnectAsync>g__WaitForConnectWithCancellation|281_0(AwaitableSocketAsyncEventArgs
+        saea, ValueTask connectTask, CancellationToken cancellationToken)\n   at System.Net.Http.HttpConnectionPool.ConnectToTcpHostAsync(String
+        host, Int32 port, HttpRequestMessage initialRequest, Boolean async, CancellationToken
+        cancellationToken)\n   --- End of inner exception stack trace ---\n   at System.Net.Http.HttpConnectionPool.ConnectToTcpHostAsync(String
+        host, Int32 port, HttpRequestMessage initialRequest, Boolean async, CancellationToken
+        cancellationToken)\n   at System.Net.Http.HttpConnectionPool.ConnectAsync(HttpRequestMessage
+        request, Boolean async, CancellationToken cancellationToken)\n   at System.Net.Http.HttpConnectionPool.CreateHttp11ConnectionAsync(HttpRequestMessage
+        request, Boolean async, CancellationToken cancellationToken)\n   at System.Net.Http.HttpConnectionPool.AddHttp11ConnectionAsync(QueueItem
+        queueItem)\n   at System.Threading.Tasks.TaskCompletionSourceWithCancellation`1.WaitWithCancellationAsync(CancellationToken
+        cancellationToken)\n   at System.Net.Http.HttpConnectionPool.HttpConnectionWaiter`1.WaitForConnectionAsync(Boolean
+        async, CancellationToken requestCancellationToken)\n   at System.Net.Http.HttpConnectionPool.SendWithVersionDetectionAndRetryAsync(HttpRequestMessage
+        request, Boolean async, Boolean doRequestAuth, CancellationToken cancellationToken)\n   at
+        System.Net.Http.RedirectHandler.SendAsync(HttpRequestMessage request, Boolean
+        async, CancellationToken cancellationToken)\n   at System.Net.Http.HttpClient.GetStreamAsyncCore(HttpRequestMessage
+        request, CancellationToken cancellationToken)\n   --- End of inner exception
+        stack trace ---\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
+        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
+        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
+        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
+        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
+        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
+        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n   ---
+        End of inner exception stack trace ---\n   at Utils.Time.Retry[T](Func`1 action,
+        Int32 maxRetries, TimeSpan retryTime, String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
+        139\n   at Core.Http.LockRetry[T](Func`1 operation, String description) in
+        /cs-codex-dist-tests/Framework/Core/Http.cs:line 212\n   at Core.Http.HttpGetStream(String
+        route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line 120\n   at CodexPlugin.CodexAccess.DownloadFile(String
+        contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line
+        71\n   at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile
+        file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
+        200\n   at CodexPlugin.CodexNode.<>c__DisplayClass35_0.<DownloadContent>b__0()
+        in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n   at
+        Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug)
+        in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n   at CodexPlugin.CodexNode.DownloadContent(ContentId
+        contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
+        128\n   at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.<DownloadTestFile>b__0()
+        in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        43\n   at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action
+        action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        82\n   at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        43\n   at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n   at
+        System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
+        invokeAttr)\n   at ContinuousTests.SingleTestRun.ThrowFailTest() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        186\n   at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        155\n   at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
+        /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Failed","testduration":"0","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T02:39:36.3628478Z","testtype":"continuous-tests"},"sort":[1707447057112]},{"_index":"continuous-tests-status-2024.02.09","_id":"bkxNi40BTe2pYqi1ySlQ","_score":null,"_ignored":["message.keyword"],"_source":{"@timestamp":"2024-02-09T00:38:55.2482106Z","avgdownload":"102.8KB/s","avgupload":"183.8KB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":" Files are not of equal length.","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"codex1-3-6c565dbd66-wxg49,bootstrap-2-75ffd985d5-sqhj5","involvedpods":"<CODEX13>,<BOOTSTRAP2>","message":"System.Exception:
+        System.Exception: Files are not of equal length.\n   at Utils.FrameworkAssert.Fail(String
+        message) in /cs-codex-dist-tests/Framework/Utils/FrameworkAssert.cs:line 12\n   at
+        Utils.FrameworkAssert.That(Boolean condition, String message) in /cs-codex-dist-tests/Framework/Utils/FrameworkAssert.cs:line
+        7\n   at FileUtils.TrackedFile.AssertEqual(TrackedFile actual) in /cs-codex-dist-tests/Framework/FileUtils/TrackedFile.cs:line
+        50\n   at FileUtils.TrackedFile.AssertIsEqual(TrackedFile actual) in /cs-codex-dist-tests/Framework/FileUtils/TrackedFile.cs:line
+        25\n   at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        45\n   at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n   at
+        System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
+        invokeAttr)\n   at ContinuousTests.SingleTestRun.ThrowFailTest() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        186\n   at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        155\n   at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
+        /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Failed","testduration":"0","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T00:21:16.3722732Z","testtype":"continuous-tests"},"sort":[1707439135248]}]}}'
+    headers:
+      Transfer-Encoding:
+      - chunked
+      X-elastic-product:
+      - Elasticsearch
+      content-type:
+      - application/vnd.elasticsearch+json;compatible-with=8
+    status:
+      code: 200
+      message: OK
+- request:
+    body: null
+    headers:
+      accept:
+      - application/vnd.elasticsearch+json; compatible-with=8
+      connection:
+      - keep-alive
+      user-agent:
+      - elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0)
+      x-elastic-client-meta:
+      - es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7
+    method: GET
+    uri: http://localhost:9200/continuous-tests-status-2024.02.09/_doc/bkxNi40BTe2pYqi1ySlQ
+  response:
+    body:
+      string: '{"_index":"continuous-tests-status-2024.02.09","_id":"bkxNi40BTe2pYqi1ySlQ","_version":1,"_seq_no":11,"_primary_term":1,"found":true,"_source":{"@timestamp":"2024-02-09T00:38:55.2482106Z","avgdownload":"102.8KB/s","avgupload":"183.8KB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":" Files are not of equal length.","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"codex1-3-6c565dbd66-wxg49,bootstrap-2-75ffd985d5-sqhj5","involvedpods":"<CODEX13>,<BOOTSTRAP2>","message":"System.Exception:
+        System.Exception: Files are not of equal length.\n   at Utils.FrameworkAssert.Fail(String
+        message) in /cs-codex-dist-tests/Framework/Utils/FrameworkAssert.cs:line 12\n   at
+        Utils.FrameworkAssert.That(Boolean condition, String message) in /cs-codex-dist-tests/Framework/Utils/FrameworkAssert.cs:line
+        7\n   at FileUtils.TrackedFile.AssertEqual(TrackedFile actual) in /cs-codex-dist-tests/Framework/FileUtils/TrackedFile.cs:line
+        50\n   at FileUtils.TrackedFile.AssertIsEqual(TrackedFile actual) in /cs-codex-dist-tests/Framework/FileUtils/TrackedFile.cs:line
+        25\n   at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        45\n   at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n   at
+        System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
+        invokeAttr)\n   at ContinuousTests.SingleTestRun.ThrowFailTest() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        186\n   at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        155\n   at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
+        /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Failed","testduration":"0","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T00:21:16.3722732Z","testtype":"continuous-tests"}}'
+    headers:
+      X-elastic-product:
+      - Elasticsearch
+      content-length:
+      - '2211'
+      content-type:
+      - application/vnd.elasticsearch+json;compatible-with=8
+    status:
+      code: 200
+      message: OK
+version: 1

logtools/resource/tests/cassettes/test_elasticsearch_log_repo/test_should_retrieve_test_runs_over_a_single_run_id.yaml

@@ -0,0 +1,287 @@
+interactions:
+- request:
+    body: '{"query":{"bool":{"filter":[{"term":{"runid.keyword":"20240208-115030"}}]}},"size":10000,"sort":[{"@timestamp":"desc"}]}'
+    headers:
+      accept:
+      - application/vnd.elasticsearch+json; compatible-with=8
+      connection:
+      - keep-alive
+      content-type:
+      - application/vnd.elasticsearch+json; compatible-with=8
+      user-agent:
+      - elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0)
+      x-elastic-client-meta:
+      - es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7
+    method: POST
+    uri: http://localhost:9200/continuous-tests-status-*/_search
+  response:
+    body:
+      string: '{"took":4,"timed_out":false,"_shards":{"total":25,"successful":25,"skipped":0,"failed":0},"hits":{"total":{"value":34,"relation":"eq"},"max_score":null,"hits":[{"_index":"continuous-tests-status-2024.02.09","_id":"VEbGi40BTe2pYqi1mE3V","_score":null,"_ignored":["message.keyword"],"_source":{"@timestamp":"2024-02-09T02:50:57.1125777Z","avgupload":"1.2MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":" data/zDvZRwzkz4EQMAECwa519qerKWmxMfeEgy4hneFg3Yr1xZZ7jHBU/network''
+        timed out after 3 tries over 10 mins, 1 secs.","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"bootstrap-2-75ffd985d5-sqhj5,codex2-4-8574cf7cdf-z75qr","involvedpods":"<BOOTSTRAP2>,<CODEX24>","message":"System.Exception:
+        System.TimeoutException: Retry ''HTTP-GET-STREAM: data/zDvZRwzkz4EQMAECwa519qerKWmxMfeEgy4hneFg3Yr1xZZ7jHBU/network''
+        timed out after 3 tries over 10 mins, 1 secs.\n ---> System.AggregateException:
+        One or more errors occurred. (One or more errors occurred. (A task was canceled.))
+        (One or more errors occurred. (A task was canceled.)) (One or more errors
+        occurred. (A task was canceled.)) (One or more errors occurred. (Connection
+        refused (codex2-4-8-int.two-client-test-bugfix.svc.cluster.local:30007)))\n
+        ---> System.AggregateException: One or more errors occurred. (A task was canceled.)\n
+        ---> System.Threading.Tasks.TaskCanceledException: A task was canceled.\n   at
+        System.Threading.Tasks.Task.GetExceptions(Boolean includeTaskCanceledExceptions)\n   at
+        System.Threading.Tasks.Task.ThrowIfExceptional(Boolean includeTaskCanceledExceptions)\n   at
+        System.Threading.Tasks.Task.Wait(Int32 millisecondsTimeout, CancellationToken
+        cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at Utils.Time.Wait[T](Task`1
+        task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0()
+        in /cs-codex-dist-tests/Framework/Core/Http.cs:line 125\n   at Utils.Time.Retry[T](Func`1
+        action, Int32 maxRetries, TimeSpan retryTime, String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
+        144\n   at Core.Http.LockRetry[T](Func`1 operation, String description) in
+        /cs-codex-dist-tests/Framework/Core/Http.cs:line 212\n   at Core.Http.HttpGetStream(String
+        route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line 120\n   at CodexPlugin.CodexAccess.DownloadFile(String
+        contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line
+        71\n   at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile
+        file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
+        200\n   at CodexPlugin.CodexNode.<>c__DisplayClass35_0.<DownloadContent>b__0()
+        in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n   at
+        Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug)
+        in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n   at CodexPlugin.CodexNode.DownloadContent(ContentId
+        contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
+        128\n   at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.<DownloadTestFile>b__0()
+        in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        43\n   at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action
+        action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        82\n   at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        43\n   at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n   at
+        System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
+        invokeAttr)\n   at System.Reflection.RuntimeMethodInfo.Invoke(Object obj,
+        BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n   at
+        ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke)
+        in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n   at
+        ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        251\n   at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        150\n   at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
+        /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n   at
+        ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.<Run>b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        53\n   at Utils.TaskFactory.CatchException(Action action, String name) in
+        /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n   at Utils.TaskFactory.<>c__DisplayClass2_0.<Run>b__0()
+        in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n   at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread
+        threadPoolThread, ExecutionContext executionContext, ContextCallback callback,
+        Object state)\n   at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task&
+        currentTaskSlot, Thread threadPoolThread)\n   at System.Threading.ThreadPoolWorkQueue.Dispatch()\n   at
+        System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n---
+        End of stack trace from previous location ---\n\n   --- End of inner exception
+        stack trace ---\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
+        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
+        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
+        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
+        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
+        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
+        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n   ---
+        End of inner exception stack trace ---\n ---> (Inner Exception #1) System.AggregateException:
+        One or more errors occurred. (A task was canceled.)\n ---> System.Threading.Tasks.TaskCanceledException:
+        A task was canceled.\n   at System.Threading.Tasks.Task.GetExceptions(Boolean
+        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
+        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
+        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
+        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
+        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
+        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
+        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n   at
+        Core.Http.LockRetry[T](Func`1 operation, String description) in /cs-codex-dist-tests/Framework/Core/Http.cs:line
+        212\n   at Core.Http.HttpGetStream(String route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line
+        120\n   at CodexPlugin.CodexAccess.DownloadFile(String contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line
+        71\n   at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile
+        file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
+        200\n   at CodexPlugin.CodexNode.<>c__DisplayClass35_0.<DownloadContent>b__0()
+        in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n   at
+        Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug)
+        in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n   at CodexPlugin.CodexNode.DownloadContent(ContentId
+        contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
+        128\n   at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.<DownloadTestFile>b__0()
+        in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        43\n   at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action
+        action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        82\n   at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        43\n   at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n   at
+        System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
+        invokeAttr)\n   at System.Reflection.RuntimeMethodInfo.Invoke(Object obj,
+        BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n   at
+        ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke)
+        in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n   at
+        ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        251\n   at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        150\n   at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
+        /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n   at
+        ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.<Run>b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        53\n   at Utils.TaskFactory.CatchException(Action action, String name) in
+        /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n   at Utils.TaskFactory.<>c__DisplayClass2_0.<Run>b__0()
+        in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n   at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread
+        threadPoolThread, ExecutionContext executionContext, ContextCallback callback,
+        Object state)\n   at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task&
+        currentTaskSlot, Thread threadPoolThread)\n   at System.Threading.ThreadPoolWorkQueue.Dispatch()\n   at
+        System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n---
+        End of stack trace from previous location ---\n\n   --- End of inner exception
+        stack trace ---\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
+        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
+        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
+        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
+        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
+        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
+        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n
+        ---> (Inner Exception #2) System.AggregateException: One or more errors occurred.
+        (A task was canceled.)\n ---> System.Threading.Tasks.TaskCanceledException:
+        A task was canceled.\n   at System.Threading.Tasks.Task.GetExceptions(Boolean
+        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
+        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
+        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
+        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
+        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
+        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
+        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n   at
+        Core.Http.LockRetry[T](Func`1 operation, String description) in /cs-codex-dist-tests/Framework/Core/Http.cs:line
+        212\n   at Core.Http.HttpGetStream(String route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line
+        120\n   at CodexPlugin.CodexAccess.DownloadFile(String contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line
+        71\n   at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile
+        file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
+        200\n   at CodexPlugin.CodexNode.<>c__DisplayClass35_0.<DownloadContent>b__0()
+        in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n   at
+        Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug)
+        in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n   at CodexPlugin.CodexNode.DownloadContent(ContentId
+        contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
+        128\n   at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.<DownloadTestFile>b__0()
+        in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        43\n   at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action
+        action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        82\n   at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        43\n   at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n   at
+        System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
+        invokeAttr)\n   at System.Reflection.RuntimeMethodInfo.Invoke(Object obj,
+        BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n   at
+        ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke)
+        in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n   at
+        ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        251\n   at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        150\n   at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
+        /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n   at
+        ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.<Run>b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        53\n   at Utils.TaskFactory.CatchException(Action action, String name) in
+        /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n   at Utils.TaskFactory.<>c__DisplayClass2_0.<Run>b__0()
+        in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n   at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread
+        threadPoolThread, ExecutionContext executionContext, ContextCallback callback,
+        Object state)\n   at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task&
+        currentTaskSlot, Thread threadPoolThread)\n   at System.Threading.ThreadPoolWorkQueue.Dispatch()\n   at
+        System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n---
+        End of stack trace from previous location ---\n\n   --- End of inner exception
+        stack trace ---\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
+        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
+        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
+        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
+        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
+        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
+        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n
+        ---> (Inner Exception #3) System.AggregateException: One or more errors occurred.
+        (Connection refused (codex2-4-8-int.two-client-test-bugfix.svc.cluster.local:30007))\n
+        ---> System.Net.Http.HttpRequestException: Connection refused (codex2-4-8-int.two-client-test-bugfix.svc.cluster.local:30007)\n
+        ---> System.Net.Sockets.SocketException (111): Connection refused\n   at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.ThrowException(SocketError
+        error, CancellationToken cancellationToken)\n   at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.System.Threading.Tasks.Sources.IValueTaskSource.GetResult(Int16
+        token)\n   at System.Net.Sockets.Socket.<ConnectAsync>g__WaitForConnectWithCancellation|281_0(AwaitableSocketAsyncEventArgs
+        saea, ValueTask connectTask, CancellationToken cancellationToken)\n   at System.Net.Http.HttpConnectionPool.ConnectToTcpHostAsync(String
+        host, Int32 port, HttpRequestMessage initialRequest, Boolean async, CancellationToken
+        cancellationToken)\n   --- End of inner exception stack trace ---\n   at System.Net.Http.HttpConnectionPool.ConnectToTcpHostAsync(String
+        host, Int32 port, HttpRequestMessage initialRequest, Boolean async, CancellationToken
+        cancellationToken)\n   at System.Net.Http.HttpConnectionPool.ConnectAsync(HttpRequestMessage
+        request, Boolean async, CancellationToken cancellationToken)\n   at System.Net.Http.HttpConnectionPool.CreateHttp11ConnectionAsync(HttpRequestMessage
+        request, Boolean async, CancellationToken cancellationToken)\n   at System.Net.Http.HttpConnectionPool.AddHttp11ConnectionAsync(QueueItem
+        queueItem)\n   at System.Threading.Tasks.TaskCompletionSourceWithCancellation`1.WaitWithCancellationAsync(CancellationToken
+        cancellationToken)\n   at System.Net.Http.HttpConnectionPool.HttpConnectionWaiter`1.WaitForConnectionAsync(Boolean
+        async, CancellationToken requestCancellationToken)\n   at System.Net.Http.HttpConnectionPool.SendWithVersionDetectionAndRetryAsync(HttpRequestMessage
+        request, Boolean async, Boolean doRequestAuth, CancellationToken cancellationToken)\n   at
+        System.Net.Http.RedirectHandler.SendAsync(HttpRequestMessage request, Boolean
+        async, CancellationToken cancellationToken)\n   at System.Net.Http.HttpClient.GetStreamAsyncCore(HttpRequestMessage
+        request, CancellationToken cancellationToken)\n   --- End of inner exception
+        stack trace ---\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
+        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
+        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
+        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
+        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
+        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
+        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n   ---
+        End of inner exception stack trace ---\n   at Utils.Time.Retry[T](Func`1 action,
+        Int32 maxRetries, TimeSpan retryTime, String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
+        139\n   at Core.Http.LockRetry[T](Func`1 operation, String description) in
+        /cs-codex-dist-tests/Framework/Core/Http.cs:line 212\n   at Core.Http.HttpGetStream(String
+        route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line 120\n   at CodexPlugin.CodexAccess.DownloadFile(String
+        contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line
+        71\n   at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile
+        file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
+        200\n   at CodexPlugin.CodexNode.<>c__DisplayClass35_0.<DownloadContent>b__0()
+        in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n   at
+        Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug)
+        in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n   at CodexPlugin.CodexNode.DownloadContent(ContentId
+        contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
+        128\n   at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.<DownloadTestFile>b__0()
+        in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        43\n   at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action
+        action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        82\n   at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        43\n   at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n   at
+        System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
+        invokeAttr)\n   at ContinuousTests.SingleTestRun.ThrowFailTest() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        186\n   at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        155\n   at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
+        /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Failed","testduration":"0","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T02:39:36.3628478Z","testtype":"continuous-tests"},"sort":[1707447057112]},{"_index":"continuous-tests-status-2024.02.09","_id":"2kK8i40BTe2pYqi1TCe9","_score":null,"_source":{"@timestamp":"2024-02-09T02:39:35.3438349Z","avgdownload":"256.2KB/s","avgupload":"2.8MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"codex2-4-8574cf7cdf-z75qr,codex1-3-6c565dbd66-wxg49","involvedpods":"<CODEX24>,<CODEX13>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"359.6048988","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T02:33:34.4148771Z","testtype":"continuous-tests"},"sort":[1707446375343]},{"_index":"continuous-tests-status-2024.02.09","_id":"whi2i40BTe2pYqi1YjlT","_score":null,"_source":{"@timestamp":"2024-02-09T02:33:09.0451640Z","avgdownload":"1.3MB/s","avgupload":"3.4MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"bootstrap-2-75ffd985d5-sqhj5,codex2-4-8574cf7cdf-z75qr","involvedpods":"<BOOTSTRAP2>,<CODEX24>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"96.8061761","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T02:31:32.1710673Z","testtype":"continuous-tests"},"sort":[1707445989045]},{"_index":"continuous-tests-status-2024.02.09","_id":"hxG0i40BTe2pYqi1iQOd","_score":null,"_source":{"@timestamp":"2024-02-09T02:31:07.9217679Z","avgdownload":"1.3MB/s","avgupload":"3.3MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"bootstrap-2-75ffd985d5-sqhj5,codex2-4-8574cf7cdf-z75qr","involvedpods":"<BOOTSTRAP2>,<CODEX24>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"96.8828193","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T02:29:30.9078520Z","testtype":"continuous-tests"},"sort":[1707445867921]},{"_index":"continuous-tests-status-2024.02.09","_id":"Bwuzi40BTe2pYqi1AMbu","_score":null,"_source":{"@timestamp":"2024-02-09T02:29:30.5454807Z","avgdownload":"269KB/s","avgupload":"3.1MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"codex2-4-8574cf7cdf-z75qr,codex1-3-6c565dbd66-wxg49","involvedpods":"<CODEX24>,<CODEX13>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"341.5041775","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T02:23:48.7157750Z","testtype":"continuous-tests"},"sort":[1707445770545]},{"_index":"continuous-tests-status-2024.02.09","_id":"gOeti40BTe2pYqi1tvXZ","_score":null,"_source":{"@timestamp":"2024-02-09T02:23:48.6094254Z","avgdownload":"162.7KB/s","avgupload":"1.9MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"bootstrap-2-75ffd985d5-sqhj5,codex1-3-6c565dbd66-wxg49","involvedpods":"<BOOTSTRAP2>,<CODEX13>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"557.2236422","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T02:14:31.1903255Z","testtype":"continuous-tests"},"sort":[1707445428609]},{"_index":"continuous-tests-status-2024.02.09","_id":"e8qli40BTe2pYqi1I_Ft","_score":null,"_source":{"@timestamp":"2024-02-09T02:14:22.5773204Z","avgdownload":"1MB/s","avgupload":"3.5MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"codex2-4-8574cf7cdf-z75qr,codex1-3-6c565dbd66-wxg49","involvedpods":"<CODEX24>,<CODEX13>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"111.0135953","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T02:12:30.6830329Z","testtype":"continuous-tests"},"sort":[1707444862577]},{"_index":"continuous-tests-status-2024.02.09","_id":"asWji40BTe2pYqi1crWx","_score":null,"_source":{"@timestamp":"2024-02-09T02:12:30.5381635Z","avgdownload":"178.9KB/s","avgupload":"2.5MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"codex2-4-8574cf7cdf-z75qr,bootstrap-2-75ffd985d5-sqhj5","involvedpods":"<CODEX24>,<BOOTSTRAP2>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"500.7759314","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T02:04:08.8279712Z","testtype":"continuous-tests"},"sort":[1707444750538]},{"_index":"continuous-tests-status-2024.02.09","_id":"raObi40BTe2pYqi1z-TG","_score":null,"_source":{"@timestamp":"2024-02-09T02:04:08.3364074Z","avgdownload":"946.6KB/s","avgupload":"2.8MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"codex1-3-6c565dbd66-wxg49,codex2-4-8574cf7cdf-z75qr","involvedpods":"<CODEX13>,<CODEX24>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"126.9885631","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T02:02:00.3407869Z","testtype":"continuous-tests"},"sort":[1707444248336]},{"_index":"continuous-tests-status-2024.02.09","_id":"FZ2Zi40BTe2pYqi1frDK","_score":null,"_source":{"@timestamp":"2024-02-09T02:01:37.7637436Z","avgdownload":"1.5MB/s","avgupload":"2.2MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"bootstrap-2-75ffd985d5-sqhj5,codex1-3-6c565dbd66-wxg49","involvedpods":"<BOOTSTRAP2>,<CODEX13>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"99.1548459","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T01:59:58.3757751Z","testtype":"continuous-tests"},"sort":[1707444097763]},{"_index":"continuous-tests-status-2024.02.09","_id":"opyXi40BTe2pYqi19gsc","_score":null,"_source":{"@timestamp":"2024-02-09T01:59:58.2293689Z","avgdownload":"401.6KB/s","avgupload":"2.7MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"codex2-4-8574cf7cdf-z75qr,bootstrap-2-75ffd985d5-sqhj5","involvedpods":"<CODEX24>,<BOOTSTRAP2>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"243.781886","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T01:55:54.2988812Z","testtype":"continuous-tests"},"sort":[1707443998229]},{"_index":"continuous-tests-status-2024.02.09","_id":"mI2Ui40BTe2pYqi1PHKr","_score":null,"_source":{"@timestamp":"2024-02-09T01:55:54.1614958Z","avgdownload":"253.7KB/s","avgupload":"3.1MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"bootstrap-2-75ffd985d5-sqhj5,codex2-4-8574cf7cdf-z75qr","involvedpods":"<BOOTSTRAP2>,<CODEX24>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"359.2512348","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T01:49:54.7221153Z","testtype":"continuous-tests"},"sort":[1707443754161]},{"_index":"continuous-tests-status-2024.02.09","_id":"Vn-Oi40BTe2pYqi1yiqP","_score":null,"_source":{"@timestamp":"2024-02-09T01:49:54.5796167Z","avgdownload":"261.5KB/s","avgupload":"3.2MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"bootstrap-2-75ffd985d5-sqhj5,codex2-4-8574cf7cdf-z75qr","involvedpods":"<BOOTSTRAP2>,<CODEX24>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"348.605598","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T01:44:04.8100487Z","testtype":"continuous-tests"},"sort":[1707443394579]},{"_index":"continuous-tests-status-2024.02.09","_id":"uXCJi40BTe2pYqi1WNZx","_score":null,"_source":{"@timestamp":"2024-02-09T01:44:04.6467851Z","avgdownload":"266.1KB/s","avgupload":"3.5MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"bootstrap-2-75ffd985d5-sqhj5,codex2-4-8574cf7cdf-z75qr","involvedpods":"<BOOTSTRAP2>,<CODEX24>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"341.4922571","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T01:38:22.8681695Z","testtype":"continuous-tests"},"sort":[1707443044646]},{"_index":"continuous-tests-status-2024.02.09","_id":"gmKEi40BTe2pYqi1NrBj","_score":null,"_source":{"@timestamp":"2024-02-09T01:38:22.7260996Z","avgdownload":"123.3KB/s","avgupload":"2.6MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"codex1-3-6c565dbd66-wxg49,bootstrap-2-75ffd985d5-sqhj5","involvedpods":"<CODEX13>,<BOOTSTRAP2>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"706.387862","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T01:26:36.1018643Z","testtype":"continuous-tests"},"sort":[1707442702726]},{"_index":"continuous-tests-status-2024.02.09","_id":"NzJ5i40BTe2pYqi1cnIN","_score":null,"_source":{"@timestamp":"2024-02-09T01:26:35.3277906Z","avgdownload":"164.2KB/s","avgupload":"1.6MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"bootstrap-2-75ffd985d5-sqhj5,codex1-3-6c565dbd66-wxg49","involvedpods":"<BOOTSTRAP2>,<CODEX13>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"560.0208629","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T01:17:15.1002846Z","testtype":"continuous-tests"},"sort":[1707441995327]},{"_index":"continuous-tests-status-2024.02.09","_id":"5hNwi40BTe2pYqi1jvqY","_score":null,"_source":{"@timestamp":"2024-02-09T01:16:53.0197926Z","avgdownload":"1.4MB/s","avgupload":"2.9MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"codex1-3-6c565dbd66-wxg49,codex2-4-8574cf7cdf-z75qr","involvedpods":"<CODEX13>,<CODEX24>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"98.2070831","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T01:15:14.6413418Z","testtype":"continuous-tests"},"sort":[1707441413019]},{"_index":"continuous-tests-status-2024.02.09","_id":"mw5vi40BTe2pYqi1BdX1","_score":null,"_source":{"@timestamp":"2024-02-09T01:15:13.6615610Z","avgdownload":"974.1KB/s","avgupload":"3MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"codex2-4-8574cf7cdf-z75qr,codex1-3-6c565dbd66-wxg49","involvedpods":"<CODEX24>,<CODEX13>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"121.6786638","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T01:13:11.3503970Z","testtype":"continuous-tests"},"sort":[1707441313661]},{"_index":"continuous-tests-status-2024.02.09","_id":"IQVsi40BTe2pYqi1tVgH","_score":null,"_source":{"@timestamp":"2024-02-09T01:12:43.7054660Z","avgdownload":"1.4MB/s","avgupload":"3.4MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"codex2-4-8574cf7cdf-z75qr,bootstrap-2-75ffd985d5-sqhj5","involvedpods":"<CODEX24>,<BOOTSTRAP2>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"92.7327451","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T01:11:10.7657168Z","testtype":"continuous-tests"},"sort":[1707441163705]},{"_index":"continuous-tests-status-2024.02.09","_id":"vP5qi40BTe2pYqi13EtB","_score":null,"_source":{"@timestamp":"2024-02-09T01:10:47.1447941Z","avgdownload":"1.2MB/s","avgupload":"3.7MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"codex2-4-8574cf7cdf-z75qr,bootstrap-2-75ffd985d5-sqhj5","involvedpods":"<CODEX24>,<BOOTSTRAP2>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"96.5233032","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T01:09:10.4700702Z","testtype":"continuous-tests"},"sort":[1707441047144]},{"_index":"continuous-tests-status-2024.02.09","_id":"hPhoi40BTe2pYqi12457","_score":null,"_source":{"@timestamp":"2024-02-09T01:08:29.9796036Z","avgdownload":"1.7MB/s","avgupload":"3.6MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"bootstrap-2-75ffd985d5-sqhj5,codex1-3-6c565dbd66-wxg49","involvedpods":"<BOOTSTRAP2>,<CODEX13>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"79.6486139","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T01:07:10.2687863Z","testtype":"continuous-tests"},"sort":[1707440909979]},{"_index":"continuous-tests-status-2024.02.09","_id":"bkxNi40BTe2pYqi1ySlQ","_score":null,"_ignored":["message.keyword"],"_source":{"@timestamp":"2024-02-09T00:38:55.2482106Z","avgdownload":"102.8KB/s","avgupload":"183.8KB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":" Files are not of equal length.","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"codex1-3-6c565dbd66-wxg49,bootstrap-2-75ffd985d5-sqhj5","involvedpods":"<CODEX13>,<BOOTSTRAP2>","message":"System.Exception:
+        System.Exception: Files are not of equal length.\n   at Utils.FrameworkAssert.Fail(String
+        message) in /cs-codex-dist-tests/Framework/Utils/FrameworkAssert.cs:line 12\n   at
+        Utils.FrameworkAssert.That(Boolean condition, String message) in /cs-codex-dist-tests/Framework/Utils/FrameworkAssert.cs:line
+        7\n   at FileUtils.TrackedFile.AssertEqual(TrackedFile actual) in /cs-codex-dist-tests/Framework/FileUtils/TrackedFile.cs:line
+        50\n   at FileUtils.TrackedFile.AssertIsEqual(TrackedFile actual) in /cs-codex-dist-tests/Framework/FileUtils/TrackedFile.cs:line
+        25\n   at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
+        45\n   at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n   at
+        System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
+        invokeAttr)\n   at ContinuousTests.SingleTestRun.ThrowFailTest() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        186\n   at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
+        155\n   at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
+        /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Failed","testduration":"0","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T00:21:16.3722732Z","testtype":"continuous-tests"},"sort":[1707439135248]},{"_index":"continuous-tests-status-2024.02.09","_id":"5PM9i40BTe2pYqi1O3cN","_score":null,"_source":{"@timestamp":"2024-02-09T00:20:47.6786305Z","avgdownload":"1.2MB/s","avgupload":"4.5MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"bootstrap-2-75ffd985d5-sqhj5,codex2-4-8574cf7cdf-z75qr","involvedpods":"<BOOTSTRAP2>,<CODEX24>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"93.0074501","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T00:19:13.8494435Z","testtype":"continuous-tests"},"sort":[1707438047678]},{"_index":"continuous-tests-status-2024.02.09","_id":"MeY7i40BTe2pYqi1OoY4","_score":null,"_source":{"@timestamp":"2024-02-09T00:18:39.0955867Z","avgdownload":"1.4MB/s","avgupload":"4.7MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"bootstrap-2-75ffd985d5-sqhj5,codex2-4-8574cf7cdf-z75qr","involvedpods":"<BOOTSTRAP2>,<CODEX24>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"85.3273743","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T00:17:13.6107407Z","testtype":"continuous-tests"},"sort":[1707437919095]},{"_index":"continuous-tests-status-2024.02.09","_id":"H9s5i40BTe2pYqi1iUKK","_score":null,"_source":{"@timestamp":"2024-02-09T00:16:46.5056103Z","avgdownload":"1.3MB/s","avgupload":"4MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"bootstrap-2-75ffd985d5-sqhj5,codex2-4-8574cf7cdf-z75qr","involvedpods":"<BOOTSTRAP2>,<CODEX24>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"92.9521056","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T00:15:13.4020376Z","testtype":"continuous-tests"},"sort":[1707437806505]},{"_index":"continuous-tests-status-2024.02.09","_id":"q843i40BTe2pYqi1iMy3","_score":null,"_source":{"@timestamp":"2024-02-09T00:14:41.6775479Z","avgdownload":"1.6MB/s","avgupload":"3MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"codex2-4-8574cf7cdf-z75qr,bootstrap-2-75ffd985d5-sqhj5","involvedpods":"<CODEX24>,<BOOTSTRAP2>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"87.2883544","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T00:13:13.1406982Z","testtype":"continuous-tests"},"sort":[1707437681677]},{"_index":"continuous-tests-status-2024.02.09","_id":"fsM1i40BTe2pYqi12HAN","_score":null,"_source":{"@timestamp":"2024-02-09T00:12:45.6542416Z","avgdownload":"1.6MB/s","avgupload":"2.6MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"codex2-4-8574cf7cdf-z75qr,bootstrap-2-75ffd985d5-sqhj5","involvedpods":"<CODEX24>,<BOOTSTRAP2>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"92.6290863","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T00:11:12.0214014Z","testtype":"continuous-tests"},"sort":[1707437565654]},{"_index":"continuous-tests-status-2024.02.09","_id":"Frczi40BTe2pYqi110M6","_score":null,"_source":{"@timestamp":"2024-02-09T00:10:38.4737224Z","avgdownload":"1.5MB/s","avgupload":"3.3MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"codex2-4-8574cf7cdf-z75qr,bootstrap-2-75ffd985d5-sqhj5","involvedpods":"<CODEX24>,<BOOTSTRAP2>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"87.3304779","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T00:09:10.0724205Z","testtype":"continuous-tests"},"sort":[1707437438473]},{"_index":"continuous-tests-status-2024.02.09","_id":"_asxi40BTe2pYqi1_n2A","_score":null,"_source":{"@timestamp":"2024-02-09T00:08:36.9681383Z","avgdownload":"1.5MB/s","avgupload":"3.7MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"codex2-4-8574cf7cdf-z75qr,bootstrap-2-75ffd985d5-sqhj5","involvedpods":"<CODEX24>,<BOOTSTRAP2>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"86.9807732","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T00:07:09.8662651Z","testtype":"continuous-tests"},"sort":[1707437316968]},{"_index":"continuous-tests-status-2024.02.09","_id":"cKUwi40BTe2pYqi1TSbm","_score":null,"_source":{"@timestamp":"2024-02-09T00:06:43.4260776Z","avgdownload":"1.3MB/s","avgupload":"3.8MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"bootstrap-2-75ffd985d5-sqhj5,codex1-3-6c565dbd66-wxg49","involvedpods":"<BOOTSTRAP2>,<CODEX13>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"93.6334484","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T00:05:09.6457592Z","testtype":"continuous-tests"},"sort":[1707437203426]},{"_index":"continuous-tests-status-2024.02.09","_id":"Np8ui40BTe2pYqi1TUoR","_score":null,"_source":{"@timestamp":"2024-02-09T00:04:35.2840789Z","avgdownload":"1.4MB/s","avgupload":"3.9MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"codex2-4-8574cf7cdf-z75qr,bootstrap-2-75ffd985d5-sqhj5","involvedpods":"<CODEX24>,<BOOTSTRAP2>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"89.4060258","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T00:03:05.7770281Z","testtype":"continuous-tests"},"sort":[1707437075284]},{"_index":"continuous-tests-status-2024.02.09","_id":"Rpksi40BTe2pYqi1dNBb","_score":null,"_source":{"@timestamp":"2024-02-09T00:02:34.8625857Z","avgdownload":"1.4MB/s","avgupload":"3.6MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"codex1-3-6c565dbd66-wxg49,bootstrap-2-75ffd985d5-sqhj5","involvedpods":"<CODEX13>,<BOOTSTRAP2>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"89.1817581","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-09T00:01:05.5565001Z","testtype":"continuous-tests"},"sort":[1707436954862]},{"_index":"continuous-tests-status-2024.02.09","_id":"Gpgqi40BTe2pYqi1dDfH","_score":null,"_source":{"@timestamp":"2024-02-09T00:00:20.1776933Z","avgdownload":"1.7MB/s","avgupload":"4.3MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"codex1-3-6c565dbd66-wxg49,codex2-4-8574cf7cdf-z75qr","involvedpods":"<CODEX13>,<CODEX24>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"76.6647678","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-08T23:59:03.4014925Z","testtype":"continuous-tests"},"sort":[1707436820177]},{"_index":"continuous-tests-status-2024.02.08","_id":"AJYoi40BTe2pYqi1Sp2i","_score":null,"_source":{"@timestamp":"2024-02-08T23:58:04.4611449Z","avgdownload":"2.4MB/s","avgupload":"4.2MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
+        build","codexrevision":"a3c95be","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpodnames":"codex1-3-6c565dbd66-wxg49,codex2-4-8574cf7cdf-z75qr","involvedpods":"<CODEX13>,<CODEX24>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240208-115030","status":"Passed","testduration":"63.1647726","testframeworkrevision":"fb00c6c","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-08T23:57:01.2110588Z","testtype":"continuous-tests"},"sort":[1707436684461]}]}}'
+    headers:
+      Transfer-Encoding:
+      - chunked
+      X-elastic-product:
+      - Elasticsearch
+      content-type:
+      - application/vnd.elasticsearch+json;compatible-with=8
+    status:
+      code: 200
+      message: OK
+version: 1

logtools/resource/tests/test_elasticsearch_log_repo.py

@@ -1,12 +1,14 @@
 import pytest
 from dateutil import parser
 
-from logtools.log.sources.input.elastic_search.elastic_search_log_repo import ElasticSearchLogRepo, Namespace, Pod
+from logtools.resource.core import Namespace, Pod
+from logtools.resource.elastic_search_log_repo import ElasticSearchLogRepo
 
 
 # XXX these are not good quality tests as they are overly complex and either tightly coupled to specific data or very
 #   weak in terms of what they assert. They will be a pain to maintain. Ideally we should build simpler fixtures and
-#   test smaller bits at a time, but that requires a lot of setup, so for now we go with this.
+#   test smaller bits at a time, but that requires a lot of setup (e.g. having pre-constructed index data in fixtures),
+#   so for now we go with this.
 
 @pytest.mark.vcr
 def test_should_retrieve_existing_namespaces():
@@ -78,3 +80,25 @@ def test_should_respect_time_horizon_for_retrieving_resources():
     namespaces = repo.namespaces('codex-continuous-tests-profiling')
 
     assert len(list(namespaces)) == 2
+
+
+@pytest.mark.vcr
+def test_should_retrieve_test_runs_over_a_single_run_id():
+    repo = ElasticSearchLogRepo()
+    runs = list(repo.test_runs('20240208-115030'))
+
+    assert len(runs) == 34
+
+
+@pytest.mark.vcr
+def test_should_retrieve_failing_test_runs_over_a_single_run_id():
+    repo = ElasticSearchLogRepo()
+    # FIXME: yes, this is a bad test. The data fixture is implicit in the recorded VCR cassette, which is neither
+    #   easy to inspect or modify.
+    runs = sorted(
+        list(repo.test_runs('20240208-115030', failed_only=True)),
+        key=lambda run: run.start)
+
+    assert len(runs) == 2
+    assert repo.test_run(runs[0].id).error.strip() == "Files are not of equal length."
+

poetry.lock

@@ -1,5 +1,36 @@
 # This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand.
 
+[[package]]
+name = "annotated-types"
+version = "0.6.0"
+description = "Reusable constraint types to use with typing.Annotated"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "annotated_types-0.6.0-py3-none-any.whl", hash = "sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43"},
+    {file = "annotated_types-0.6.0.tar.gz", hash = "sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d"},
+]
+
+[[package]]
+name = "anyio"
+version = "4.2.0"
+description = "High level compatibility layer for multiple asynchronous event loop implementations"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "anyio-4.2.0-py3-none-any.whl", hash = "sha256:745843b39e829e108e518c489b31dc757de7d2131d53fac32bd8df268227bfee"},
+    {file = "anyio-4.2.0.tar.gz", hash = "sha256:e1875bb4b4e2de1669f4bc7869b6d3f54231cdced71605e6e64c9be77e3be50f"},
+]
+
+[package.dependencies]
+idna = ">=2.8"
+sniffio = ">=1.1"
+
+[package.extras]
+doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"]
+test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"]
+trio = ["trio (>=0.23)"]
+
 [[package]]
 name = "appnope"
 version = "0.1.3"
@@ -133,6 +164,25 @@ files = [
 [package.extras]
 tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich"]
 
+[[package]]
+name = "fastapi"
+version = "0.109.0"
+description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "fastapi-0.109.0-py3-none-any.whl", hash = "sha256:8c77515984cd8e8cfeb58364f8cc7a28f0692088475e2614f7bf03275eba9093"},
+    {file = "fastapi-0.109.0.tar.gz", hash = "sha256:b978095b9ee01a5cf49b19f4bc1ac9b8ca83aa076e770ef8fd9af09a2b88d191"},
+]
+
+[package.dependencies]
+pydantic = ">=1.7.4,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0 || >2.0.0,<2.0.1 || >2.0.1,<2.1.0 || >2.1.0,<3.0.0"
+starlette = ">=0.35.0,<0.36.0"
+typing-extensions = ">=4.8.0"
+
+[package.extras]
+all = ["email-validator (>=2.0.0)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=2.11.2)", "orjson (>=3.2.1)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.5)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"]
+
 [[package]]
 name = "idna"
 version = "3.4"
@@ -223,6 +273,30 @@ docs = ["Jinja2 (==2.11.3)", "MarkupSafe (==1.1.1)", "Pygments (==2.8.1)", "alab
 qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"]
 testing = ["Django", "attrs", "colorama", "docopt", "pytest (<7.0.0)"]
 
+[[package]]
+name = "markdown-it-py"
+version = "3.0.0"
+description = "Python port of markdown-it. Markdown parsing, done right!"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"},
+    {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"},
+]
+
+[package.dependencies]
+mdurl = ">=0.1,<1.0"
+
+[package.extras]
+benchmarking = ["psutil", "pytest", "pytest-benchmark"]
+code-style = ["pre-commit (>=3.0,<4.0)"]
+compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"]
+linkify = ["linkify-it-py (>=1,<3)"]
+plugins = ["mdit-py-plugins"]
+profiling = ["gprof2dot"]
+rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"]
+testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"]
+
 [[package]]
 name = "matplotlib-inline"
 version = "0.1.6"
@@ -237,6 +311,17 @@ files = [
 [package.dependencies]
 traitlets = "*"
 
+[[package]]
+name = "mdurl"
+version = "0.1.2"
+description = "Markdown URL utilities"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"},
+    {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"},
+]
+
 [[package]]
 name = "multidict"
 version = "6.0.4"
@@ -442,23 +527,6 @@ files = [
 dev = ["pre-commit", "tox"]
 testing = ["pytest", "pytest-benchmark"]
 
-[[package]]
-name = "prettytable"
-version = "3.9.0"
-description = "A simple Python library for easily displaying tabular data in a visually appealing ASCII table format"
-optional = false
-python-versions = ">=3.8"
-files = [
-    {file = "prettytable-3.9.0-py3-none-any.whl", hash = "sha256:a71292ab7769a5de274b146b276ce938786f56c31cf7cea88b6f3775d82fe8c8"},
-    {file = "prettytable-3.9.0.tar.gz", hash = "sha256:f4ed94803c23073a90620b201965e5dc0bccf1760b7a7eaf3158cab8aaffdf34"},
-]
-
-[package.dependencies]
-wcwidth = "*"
-
-[package.extras]
-tests = ["pytest", "pytest-cov", "pytest-lazy-fixture"]
-
 [[package]]
 name = "prompt-toolkit"
 version = "3.0.39"
@@ -498,6 +566,116 @@ files = [
 [package.extras]
 tests = ["pytest"]
 
+[[package]]
+name = "pydantic"
+version = "2.6.0"
+description = "Data validation using Python type hints"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "pydantic-2.6.0-py3-none-any.whl", hash = "sha256:1440966574e1b5b99cf75a13bec7b20e3512e8a61b894ae252f56275e2c465ae"},
+    {file = "pydantic-2.6.0.tar.gz", hash = "sha256:ae887bd94eb404b09d86e4d12f93893bdca79d766e738528c6fa1c849f3c6bcf"},
+]
+
+[package.dependencies]
+annotated-types = ">=0.4.0"
+pydantic-core = "2.16.1"
+typing-extensions = ">=4.6.1"
+
+[package.extras]
+email = ["email-validator (>=2.0.0)"]
+
+[[package]]
+name = "pydantic-core"
+version = "2.16.1"
+description = ""
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "pydantic_core-2.16.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:300616102fb71241ff477a2cbbc847321dbec49428434a2f17f37528721c4948"},
+    {file = "pydantic_core-2.16.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5511f962dd1b9b553e9534c3b9c6a4b0c9ded3d8c2be96e61d56f933feef9e1f"},
+    {file = "pydantic_core-2.16.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:98f0edee7ee9cc7f9221af2e1b95bd02810e1c7a6d115cfd82698803d385b28f"},
+    {file = "pydantic_core-2.16.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9795f56aa6b2296f05ac79d8a424e94056730c0b860a62b0fdcfe6340b658cc8"},
+    {file = "pydantic_core-2.16.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c45f62e4107ebd05166717ac58f6feb44471ed450d07fecd90e5f69d9bf03c48"},
+    {file = "pydantic_core-2.16.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:462d599299c5971f03c676e2b63aa80fec5ebc572d89ce766cd11ca8bcb56f3f"},
+    {file = "pydantic_core-2.16.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21ebaa4bf6386a3b22eec518da7d679c8363fb7fb70cf6972161e5542f470798"},
+    {file = "pydantic_core-2.16.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:99f9a50b56713a598d33bc23a9912224fc5d7f9f292444e6664236ae471ddf17"},
+    {file = "pydantic_core-2.16.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:8ec364e280db4235389b5e1e6ee924723c693cbc98e9d28dc1767041ff9bc388"},
+    {file = "pydantic_core-2.16.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:653a5dfd00f601a0ed6654a8b877b18d65ac32c9d9997456e0ab240807be6cf7"},
+    {file = "pydantic_core-2.16.1-cp310-none-win32.whl", hash = "sha256:1661c668c1bb67b7cec96914329d9ab66755911d093bb9063c4c8914188af6d4"},
+    {file = "pydantic_core-2.16.1-cp310-none-win_amd64.whl", hash = "sha256:561be4e3e952c2f9056fba5267b99be4ec2afadc27261505d4992c50b33c513c"},
+    {file = "pydantic_core-2.16.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:102569d371fadc40d8f8598a59379c37ec60164315884467052830b28cc4e9da"},
+    {file = "pydantic_core-2.16.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:735dceec50fa907a3c314b84ed609dec54b76a814aa14eb90da31d1d36873a5e"},
+    {file = "pydantic_core-2.16.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e83ebbf020be727d6e0991c1b192a5c2e7113eb66e3def0cd0c62f9f266247e4"},
+    {file = "pydantic_core-2.16.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:30a8259569fbeec49cfac7fda3ec8123486ef1b729225222f0d41d5f840b476f"},
+    {file = "pydantic_core-2.16.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:920c4897e55e2881db6a6da151198e5001552c3777cd42b8a4c2f72eedc2ee91"},
+    {file = "pydantic_core-2.16.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f5247a3d74355f8b1d780d0f3b32a23dd9f6d3ff43ef2037c6dcd249f35ecf4c"},
+    {file = "pydantic_core-2.16.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d5bea8012df5bb6dda1e67d0563ac50b7f64a5d5858348b5c8cb5043811c19d"},
+    {file = "pydantic_core-2.16.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ed3025a8a7e5a59817b7494686d449ebfbe301f3e757b852c8d0d1961d6be864"},
+    {file = "pydantic_core-2.16.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:06f0d5a1d9e1b7932477c172cc720b3b23c18762ed7a8efa8398298a59d177c7"},
+    {file = "pydantic_core-2.16.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:150ba5c86f502c040b822777e2e519b5625b47813bd05f9273a8ed169c97d9ae"},
+    {file = "pydantic_core-2.16.1-cp311-none-win32.whl", hash = "sha256:d6cbdf12ef967a6aa401cf5cdf47850559e59eedad10e781471c960583f25aa1"},
+    {file = "pydantic_core-2.16.1-cp311-none-win_amd64.whl", hash = "sha256:afa01d25769af33a8dac0d905d5c7bb2d73c7c3d5161b2dd6f8b5b5eea6a3c4c"},
+    {file = "pydantic_core-2.16.1-cp311-none-win_arm64.whl", hash = "sha256:1a2fe7b00a49b51047334d84aafd7e39f80b7675cad0083678c58983662da89b"},
+    {file = "pydantic_core-2.16.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:0f478ec204772a5c8218e30eb813ca43e34005dff2eafa03931b3d8caef87d51"},
+    {file = "pydantic_core-2.16.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f1936ef138bed2165dd8573aa65e3095ef7c2b6247faccd0e15186aabdda7f66"},
+    {file = "pydantic_core-2.16.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99d3a433ef5dc3021c9534a58a3686c88363c591974c16c54a01af7efd741f13"},
+    {file = "pydantic_core-2.16.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bd88f40f2294440d3f3c6308e50d96a0d3d0973d6f1a5732875d10f569acef49"},
+    {file = "pydantic_core-2.16.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3fac641bbfa43d5a1bed99d28aa1fded1984d31c670a95aac1bf1d36ac6ce137"},
+    {file = "pydantic_core-2.16.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:72bf9308a82b75039b8c8edd2be2924c352eda5da14a920551a8b65d5ee89253"},
+    {file = "pydantic_core-2.16.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb4363e6c9fc87365c2bc777a1f585a22f2f56642501885ffc7942138499bf54"},
+    {file = "pydantic_core-2.16.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:20f724a023042588d0f4396bbbcf4cffd0ddd0ad3ed4f0d8e6d4ac4264bae81e"},
+    {file = "pydantic_core-2.16.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:fb4370b15111905bf8b5ba2129b926af9470f014cb0493a67d23e9d7a48348e8"},
+    {file = "pydantic_core-2.16.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:23632132f1fd608034f1a56cc3e484be00854db845b3a4a508834be5a6435a6f"},
+    {file = "pydantic_core-2.16.1-cp312-none-win32.whl", hash = "sha256:b9f3e0bffad6e238f7acc20c393c1ed8fab4371e3b3bc311020dfa6020d99212"},
+    {file = "pydantic_core-2.16.1-cp312-none-win_amd64.whl", hash = "sha256:a0b4cfe408cd84c53bab7d83e4209458de676a6ec5e9c623ae914ce1cb79b96f"},
+    {file = "pydantic_core-2.16.1-cp312-none-win_arm64.whl", hash = "sha256:d195add190abccefc70ad0f9a0141ad7da53e16183048380e688b466702195dd"},
+    {file = "pydantic_core-2.16.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:502c062a18d84452858f8aea1e520e12a4d5228fc3621ea5061409d666ea1706"},
+    {file = "pydantic_core-2.16.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d8c032ccee90b37b44e05948b449a2d6baed7e614df3d3f47fe432c952c21b60"},
+    {file = "pydantic_core-2.16.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:920f4633bee43d7a2818e1a1a788906df5a17b7ab6fe411220ed92b42940f818"},
+    {file = "pydantic_core-2.16.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9f5d37ff01edcbace53a402e80793640c25798fb7208f105d87a25e6fcc9ea06"},
+    {file = "pydantic_core-2.16.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:399166f24c33a0c5759ecc4801f040dbc87d412c1a6d6292b2349b4c505effc9"},
+    {file = "pydantic_core-2.16.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ac89ccc39cd1d556cc72d6752f252dc869dde41c7c936e86beac5eb555041b66"},
+    {file = "pydantic_core-2.16.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73802194f10c394c2bedce7a135ba1d8ba6cff23adf4217612bfc5cf060de34c"},
+    {file = "pydantic_core-2.16.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8fa00fa24ffd8c31fac081bf7be7eb495be6d248db127f8776575a746fa55c95"},
+    {file = "pydantic_core-2.16.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:601d3e42452cd4f2891c13fa8c70366d71851c1593ed42f57bf37f40f7dca3c8"},
+    {file = "pydantic_core-2.16.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:07982b82d121ed3fc1c51faf6e8f57ff09b1325d2efccaa257dd8c0dd937acca"},
+    {file = "pydantic_core-2.16.1-cp38-none-win32.whl", hash = "sha256:d0bf6f93a55d3fa7a079d811b29100b019784e2ee6bc06b0bb839538272a5610"},
+    {file = "pydantic_core-2.16.1-cp38-none-win_amd64.whl", hash = "sha256:fbec2af0ebafa57eb82c18c304b37c86a8abddf7022955d1742b3d5471a6339e"},
+    {file = "pydantic_core-2.16.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:a497be217818c318d93f07e14502ef93d44e6a20c72b04c530611e45e54c2196"},
+    {file = "pydantic_core-2.16.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:694a5e9f1f2c124a17ff2d0be613fd53ba0c26de588eb4bdab8bca855e550d95"},
+    {file = "pydantic_core-2.16.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d4dfc66abea3ec6d9f83e837a8f8a7d9d3a76d25c9911735c76d6745950e62c"},
+    {file = "pydantic_core-2.16.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8655f55fe68c4685673265a650ef71beb2d31871c049c8b80262026f23605ee3"},
+    {file = "pydantic_core-2.16.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:21e3298486c4ea4e4d5cc6fb69e06fb02a4e22089304308817035ac006a7f506"},
+    {file = "pydantic_core-2.16.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:71b4a48a7427f14679f0015b13c712863d28bb1ab700bd11776a5368135c7d60"},
+    {file = "pydantic_core-2.16.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10dca874e35bb60ce4f9f6665bfbfad050dd7573596608aeb9e098621ac331dc"},
+    {file = "pydantic_core-2.16.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fa496cd45cda0165d597e9d6f01e36c33c9508f75cf03c0a650018c5048f578e"},
+    {file = "pydantic_core-2.16.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5317c04349472e683803da262c781c42c5628a9be73f4750ac7d13040efb5d2d"},
+    {file = "pydantic_core-2.16.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:42c29d54ed4501a30cd71015bf982fa95e4a60117b44e1a200290ce687d3e640"},
+    {file = "pydantic_core-2.16.1-cp39-none-win32.whl", hash = "sha256:ba07646f35e4e49376c9831130039d1b478fbfa1215ae62ad62d2ee63cf9c18f"},
+    {file = "pydantic_core-2.16.1-cp39-none-win_amd64.whl", hash = "sha256:2133b0e412a47868a358713287ff9f9a328879da547dc88be67481cdac529118"},
+    {file = "pydantic_core-2.16.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:d25ef0c33f22649b7a088035fd65ac1ce6464fa2876578df1adad9472f918a76"},
+    {file = "pydantic_core-2.16.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:99c095457eea8550c9fa9a7a992e842aeae1429dab6b6b378710f62bfb70b394"},
+    {file = "pydantic_core-2.16.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b49c604ace7a7aa8af31196abbf8f2193be605db6739ed905ecaf62af31ccae0"},
+    {file = "pydantic_core-2.16.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c56da23034fe66221f2208c813d8aa509eea34d97328ce2add56e219c3a9f41c"},
+    {file = "pydantic_core-2.16.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cebf8d56fee3b08ad40d332a807ecccd4153d3f1ba8231e111d9759f02edfd05"},
+    {file = "pydantic_core-2.16.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:1ae8048cba95f382dba56766525abca438328455e35c283bb202964f41a780b0"},
+    {file = "pydantic_core-2.16.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:780daad9e35b18d10d7219d24bfb30148ca2afc309928e1d4d53de86822593dc"},
+    {file = "pydantic_core-2.16.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c94b5537bf6ce66e4d7830c6993152940a188600f6ae044435287753044a8fe2"},
+    {file = "pydantic_core-2.16.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:adf28099d061a25fbcc6531febb7a091e027605385de9fe14dd6a97319d614cf"},
+    {file = "pydantic_core-2.16.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:644904600c15816a1f9a1bafa6aab0d21db2788abcdf4e2a77951280473f33e1"},
+    {file = "pydantic_core-2.16.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:87bce04f09f0552b66fca0c4e10da78d17cb0e71c205864bab4e9595122cb9d9"},
+    {file = "pydantic_core-2.16.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:877045a7969ace04d59516d5d6a7dee13106822f99a5d8df5e6822941f7bedc8"},
+    {file = "pydantic_core-2.16.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9c46e556ee266ed3fb7b7a882b53df3c76b45e872fdab8d9cf49ae5e91147fd7"},
+    {file = "pydantic_core-2.16.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:4eebbd049008eb800f519578e944b8dc8e0f7d59a5abb5924cc2d4ed3a1834ff"},
+    {file = "pydantic_core-2.16.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:c0be58529d43d38ae849a91932391eb93275a06b93b79a8ab828b012e916a206"},
+    {file = "pydantic_core-2.16.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:b1fc07896fc1851558f532dffc8987e526b682ec73140886c831d773cef44b76"},
+    {file = "pydantic_core-2.16.1.tar.gz", hash = "sha256:daff04257b49ab7f4b3f73f98283d3dbb1a65bf3500d55c7beac3c66c310fe34"},
+]
+
+[package.dependencies]
+typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"
+
 [[package]]
 name = "pygments"
 version = "2.16.1"
@@ -635,6 +813,24 @@ files = [
     {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"},
 ]
 
+[[package]]
+name = "rich"
+version = "13.7.0"
+description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal"
+optional = false
+python-versions = ">=3.7.0"
+files = [
+    {file = "rich-13.7.0-py3-none-any.whl", hash = "sha256:6da14c108c4866ee9520bbffa71f6fe3962e193b7da68720583850cd4548e235"},
+    {file = "rich-13.7.0.tar.gz", hash = "sha256:5cb5123b5cf9ee70584244246816e9114227e0b98ad9176eede6ad54bf5403fa"},
+]
+
+[package.dependencies]
+markdown-it-py = ">=2.2.0"
+pygments = ">=2.13.0,<3.0.0"
+
+[package.extras]
+jupyter = ["ipywidgets (>=7.5.1,<9)"]
+
 [[package]]
 name = "six"
 version = "1.16.0"
@@ -646,6 +842,17 @@ files = [
     {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
 ]
 
+[[package]]
+name = "sniffio"
+version = "1.3.0"
+description = "Sniff out which async library your code is running under"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"},
+    {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"},
+]
+
 [[package]]
 name = "stack-data"
 version = "0.6.3"
@@ -665,6 +872,23 @@ pure-eval = "*"
 [package.extras]
 tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"]
 
+[[package]]
+name = "starlette"
+version = "0.35.1"
+description = "The little ASGI library that shines."
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "starlette-0.35.1-py3-none-any.whl", hash = "sha256:50bbbda9baa098e361f398fda0928062abbaf1f54f4fadcbe17c092a01eb9a25"},
+    {file = "starlette-0.35.1.tar.gz", hash = "sha256:3e2639dac3520e4f58734ed22553f950d3f3cb1001cd2eaac4d57e8cdc5f66bc"},
+]
+
+[package.dependencies]
+anyio = ">=3.4.0,<5"
+
+[package.extras]
+full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart", "pyyaml"]
+
 [[package]]
 name = "traitlets"
 version = "5.11.2"
@@ -925,5 +1149,5 @@ multidict = ">=4.0"
 
 [metadata]
 lock-version = "2.0"
-python-versions = "~3.11"
-content-hash = "ed54352d543c77435a48e65debaf1005c58f524f18ba627e9a9c956ea77e2b91"
+python-versions = ">=3.11,<3.12"
+content-hash = "4362ec2eb2a1d5b64b491a0695ac2aa0509d7779a16fdbfd0e2242a3239497bc"

pyproject.toml

@@ -1,17 +1,18 @@
 [tool.poetry]
 name = "logtools"
-version = "1.0.1"
+version = "1.4.2"
 description = ""
 authors = ["gmega <giuliano@status.im>"]
 readme = "README.md"
 
 [tool.poetry.dependencies]
-python = "~3.11"
+python = ">=3.11,<3.12"
 pytz = "^2023.3.post1"
 colored = "^2.2.3"
 python-dateutil = "^2.8.2"
 elasticsearch = "^8.10.1"
-prettytable = "^3.9.0"
+fastapi = "^0.109.0"
+rich = "^13.7.0"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.4.2"