allow retrieval of logs from individual test runs

logtools/cli/es_logs.py

@@ -4,9 +4,8 @@ from argparse import ArgumentParser
 from datetime import timedelta, datetime
 from enum import Enum
 from json import JSONEncoder
-from typing import List, Iterable, Any
+from typing import List, Iterable, Any, Optional, Set
 
-import rich
 from colored import Style
 from dateutil import parser as tsparser
 from elasticsearch import Elasticsearch
@@ -26,17 +25,79 @@ class ResourceType(Enum):
     runs = 'runs'
 
 
-GETTERS = {
+RESOURCE_GETTERS = {
     ResourceType.pods: lambda repo, args: repo.pods(prefix=args.prefix, run_id=args.run_id),
     ResourceType.namespaces: lambda repo, args: repo.namespaces(prefix=args.prefix),
     ResourceType.runs: lambda repo, args: repo.test_runs(run_id=args.run_id, failed_only=args.failed_only),
 }
 
-DESCRIBERS = {
+RESOURCE_DESCRIBERS = {
-    ResourceType.runs: lambda repo, args: repo.describe_test_run(test_run_id=args.test_run_id),
+    ResourceType.runs: lambda repo, args: repo.test_run(test_run_id=args.test_run_id),
 }
 
 
+def main():
+    parser = ArgumentParser()
+    parser.add_argument(
+        '--es-host',
+        help='ElasticSearch URL (defaults to http://localhost:9200)',
+        default=os.environ.get('ES_HOST', 'http://localhost:9200')
+    )
+
+    subparsers = parser.add_subparsers(title='Command', required=True)
+    _add_get_cli(subparsers)
+    _add_describe_cli(subparsers)
+    _add_logs_cli(subparsers)
+
+    args = parser.parse_args()
+    client = Elasticsearch(args.es_host, request_timeout=60)
+    args.main(args, client)
+
+
+def get_object(args, client: Elasticsearch):
+    repo = ElasticSearchLogRepo(client=client)
+    Console().print(format_table(RESOURCE_GETTERS[ResourceType[args.resource_type]](repo, args)))
+
+
+def describe_object(args, client: Elasticsearch):
+    repo = ElasticSearchLogRepo(client=client)
+    Console().print(format_json([RESOURCE_DESCRIBERS[ResourceType[args.resource_type]](repo, args)]))
+
+
+# FIXME this is starting to get too complex to be here.
+def get_logs(args, client: Elasticsearch):
+    resource = ResourceType[args.resource_type]
+    if resource == ResourceType.pods:
+        get_pod_logs(
+            pods=args.pods,
+            client=client,
+            colored_output=not args.no_color,
+            start_date=args.from_,
+            end_date=args.to,
+        )
+    elif resource == ResourceType.runs:
+        run = ElasticSearchLogRepo(client=client).test_run(test_run_id=args.test_run_id).test_run
+        get_pod_logs(set(run.pods), client, start_date=run.start, end_date=run.end)
+
+
+def get_pod_logs(pods: Set[str],
+                 client: Elasticsearch,
+                 colored_output: bool = True,
+                 start_date: Optional[datetime] = None,
+                 end_date: Optional[datetime] = None):
+    colors = ColorMap()
+    for line in ElasticSearchSource(
+            pods=pods,
+            client=client,
+            start_date=start_date,
+            end_date=end_date,
+    ):
+        output = f'[{line.location.pod_name}]: {line.raw}'
+        if colored_output:
+            output = f'{colors[line.location.pod_name]}{output}{Style.reset}'
+        print(output)
+
+
 def format_table(objects: List, title: str = 'Results') -> Table:
     tbl = Table(title=title)
     field_names = None
@@ -75,95 +136,60 @@ def _format_field(field: Any):
     return str(field)
 
 
-def get_object(args, client: Elasticsearch):
-    repo = ElasticSearchLogRepo(client=client)
-    Console().print(format_table(GETTERS[ResourceType[args.resource_type]](repo, args)))
-
-
-def describe_object(args, client: Elasticsearch):
-    repo = ElasticSearchLogRepo(client=client)
-    Console().print(format_json([DESCRIBERS[ResourceType[args.resource_type]](repo, args)]))
-
-
-def get_logs(args, client: Elasticsearch):
-    colors = ColorMap()
-    for line in ElasticSearchSource(
-            pods=args.pods,
-            client=client,
-            start_date=args.from_,
-            end_date=args.to,
-    ):
-        output = f'[{line.location.pod_name}]: {line.raw}'
-        if not args.no_color:
-            output = f'{colors[line.location.pod_name]}{output}{Style.reset}'
-        print(output)
-
-
-def main():
-    parser = ArgumentParser()
-    parser.add_argument(
-        '--es-host',
-        help='ElasticSearch URL (defaults to http://localhost:9200)',
-        default=os.environ.get('ES_HOST', 'http://localhost:9200')
-    )
-
-    subparsers = parser.add_subparsers(title='Command', required=True)
-    _add_get_cli(subparsers)
-    _add_describe_cli(subparsers)
-    _add_logs_cli(subparsers)
-
-    args = parser.parse_args()
-    client = Elasticsearch(args.es_host, request_timeout=60)
-    args.main(args, client)
-
-
 def _add_get_cli(subparsers):
-    get = subparsers.add_parser('get', help='Display existing resources')
+    get = subparsers.add_parser('get', help='display existing resources')
     get.add_argument('--from', type=tsparser.parse,
-                     help='Show resources present in log messages starting at the given date '
+                     help='show resources present in log messages starting at the given date '
-                          '(MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm). Defaults to 7 days ago.',
+                          '(mm-dd-yyyy, or mm-dd-yyyy hh:mm:ss.mmmmmm). defaults to 7 days ago.',
                      default=(datetime.today() - timedelta(days=7)).date())
     get.set_defaults(main=get_object)
 
-    get_subparsers = get.add_subparsers(title='Resource type', dest='resource_type', required=True)
+    get_subparsers = get.add_subparsers(title='resource type', dest='resource_type', required=True)
-    get_pods = get_subparsers.add_parser('pods', help='Display existing pods')
+    get_pods = get_subparsers.add_parser('pods', help='display existing pods')
-    get_pods.add_argument('--prefix', help='Filter pods by prefix')
+    get_pods.add_argument('--prefix', help='filter pods by prefix')
-    get_pods.add_argument('--run-id', help='Show pods for a given run', required=True)
+    get_pods.add_argument('--run-id', help='show pods for a given run', required=True)
 
-    get_namespaces = get_subparsers.add_parser('namespaces', help='Display existing namespaces')
+    get_namespaces = get_subparsers.add_parser('namespaces', help='display existing namespaces')
-    get_namespaces.add_argument('--prefix', help='Filter namespaces by prefix')
+    get_namespaces.add_argument('--prefix', help='filter namespaces by prefix')
 
-    get_namespaces = get_subparsers.add_parser('runs', help='Display current test runs')
+    get_namespaces = get_subparsers.add_parser('runs', help='display current test runs')
-    get_namespaces.add_argument('--run-id', help='Show test runs for the given run id', required=True)
+    get_namespaces.add_argument('--run-id', help='show test runs for the given run id', required=True)
-    get_namespaces.add_argument('--failed-only', action='store_true', help='Show only failed test runs')
+    get_namespaces.add_argument('--failed-only', action='store_true', help='show only failed test runs')
     get_namespaces.add_argument('--from', type=tsparser.parse,
-                                help='Show test runs starting at the given date '
+                                help='show test runs starting at the given date '
-                                     '(MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm). Defaults to 7 days ago.',
+                                     '(mm-dd-yyyy, or mm-dd-yyyy hh:mm:ss.mmmmmm). defaults to 7 days ago.',
                                 default=(datetime.today() - timedelta(days=7)).date())
 
 
 def _add_describe_cli(subparsers):
-    describe = subparsers.add_parser('describe', help='Describe a resource')
+    describe = subparsers.add_parser('describe', help='describe a resource')
     describe.set_defaults(main=describe_object)
 
-    describe_subparsers = describe.add_subparsers(title='Resource type', dest='resource_type', required=True)
+    describe_subparsers = describe.add_subparsers(title='resource type', dest='resource_type', required=True)
-    describe_runs = describe_subparsers.add_parser('runs', help='Describe a test run')
+    describe_runs = describe_subparsers.add_parser('runs', help='describe a test run')
-    describe_runs.add_argument('test_run_id', help='Show test run details')
+    describe_runs.add_argument('test_run_id', help='show test run details')
     describe_runs.set_defaults(main=describe_object)
 
 
 def _add_logs_cli(subparsers):
-    logs = subparsers.add_parser('logs', help='Fetch pod logs')
+    logs = subparsers.add_parser('logs', help='fetch pod logs')
     logs.set_defaults(main=get_logs)
 
-    logs.add_argument('--pods', nargs='+', help='Pods to fetch logs for', required=True)
+    log_subparsers = logs.add_subparsers(title='resource type', dest='resource_type', required=True)
-    logs.add_argument('--from', dest='from_', type=tsparser.parse,
+
-                      help='Show entries from date/time (MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm), '
+    logs.add_argument('--no-color', dest='no_color', action='store_true', help='disable colored output')
+
+    pod_logs = log_subparsers.add_parser('pods', help='fetch logs for a pod')
+    pod_logs.add_argument('pods', nargs='+', help='pod names to fetch logs from')
+    pod_logs.add_argument('--from', dest='from_', type=tsparser.parse,
+                      help='show entries from date/time (MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm), '
                            'treated as UTC if no timezone given', default=None)
-    logs.add_argument('--to', dest='to', type=tsparser.parse,
+    pod_logs.add_argument('--to', dest='to', type=tsparser.parse,
-                      help='Show entries until date/time (MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm), '
+                      help='show entries until date/time (MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm), '
                            'treated as UTC if no timezone given', default=None)
-    logs.add_argument('--no-color', dest='no_color', action='store_true', help='Disable colored output')
+
+    run_logs = log_subparsers.add_parser('runs', help='fetch logs for a test run')
+    run_logs.add_argument('test_run_id', help='run ID to fetch logs from')
 
 
 if __name__ == '__main__':

logtools/resource/core.py

@@ -2,7 +2,7 @@ import abc
 from dataclasses import dataclass
 from datetime import datetime
 from enum import Enum
-from typing import Iterator, Optional
+from typing import Iterator, Optional, List
 
 
 @dataclass(frozen=True)
@@ -26,11 +26,11 @@ class TestStatus(Enum):
 
 
 @dataclass(frozen=True)
-class TestRun:
+class SummarizedTestRun:
     id: str
     run_id: str
     test_name: str
-    pods: str
+    pods: List[str]
     start: datetime
     end: datetime
     duration: float
@@ -38,8 +38,8 @@ class TestRun:
 
 
 @dataclass(frozen=True)
-class TestRunDescription:
+class TestRun:
-    test_run: TestRun
+    test_run: SummarizedTestRun
     error: Optional[str]
     stacktrace: Optional[str]
 
@@ -54,9 +54,9 @@ class Repository(abc.ABC):
         ...
 
     @abc.abstractmethod
-    def test_runs(self, run_id: str, failed_only=False) -> Iterator[TestRun]:
+    def test_runs(self, run_id: str, failed_only=False) -> Iterator[SummarizedTestRun]:
         ...
 
     @abc.abstractmethod
-    def describe_test_run(self, test_run_id: str) -> TestRunDescription:
+    def test_run(self, test_run_id: str) -> TestRun:
         ...

logtools/resource/elastic_search_log_repo.py

@@ -5,7 +5,7 @@ from typing import Optional, Iterator, Dict, Any
 from dateutil import parser
 from elasticsearch import Elasticsearch
 
-from logtools.resource.core import Repository, TestRunDescription, TestStatus, TestRun, Namespace, Pod
+from logtools.resource.core import Repository, TestRun, TestStatus, SummarizedTestRun, Namespace, Pod
 
 logger = logging.getLogger(__name__)
 
@@ -92,7 +92,7 @@ class ElasticSearchLogRepo(Repository):
                 indices=tuple(sorted(index['key'] for index in pod['indices']['buckets']))
             )
 
-    def test_runs(self, run_id: str, failed_only=False) -> Iterator[TestRun]:
+    def test_runs(self, run_id: str, failed_only=False) -> Iterator[SummarizedTestRun]:
         query = self._time_limited({
             'query': {
                 'bool': {
@@ -109,21 +109,21 @@ class ElasticSearchLogRepo(Repository):
         for document in self.client.search(index=TEST_STATUS_INDEX_SET, body=query)['hits']['hits']:  # type: ignore
             yield self._test_run_from_document(document['_index'], document['_id'], document['_source'])
 
-    def describe_test_run(self, test_run_id: str) -> TestRunDescription:
+    def test_run(self, test_run_id: str) -> TestRun:
         index, doc_id = test_run_id.split('/')
         document = self.client.get(index=index, id=doc_id)
         source = document['_source']
-        return TestRunDescription(
+        return TestRun(
             test_run=self._test_run_from_document(document['_index'], document['_id'], source),
             error=source.get('error'),
             stacktrace=source.get('message'),
         )
 
     @staticmethod
-    def _test_run_from_document(index: str, doc_id: str, source: Dict[str, str]):
+    def _test_run_from_document(index: str, doc_id: str, source: Dict[str, str]) -> SummarizedTestRun:
         start = parser.parse(source['teststart'])
         duration = float(source['testduration'])
-        return TestRun(
+        return SummarizedTestRun(
             id=f"{index}/{doc_id}",
             run_id=source['runid'],
             test_name=source['testname'],
@@ -131,7 +131,7 @@ class ElasticSearchLogRepo(Repository):
             end=start + timedelta(seconds=duration),
             duration=duration,
             status=TestStatus(source['status'].lower()),
-            pods=source['involvedpods'],
+            pods=source['involvedpodnames'].split(','),
         )
 
     def _time_limited(self, query: Dict[str, Any]) -> Dict[str, Any]:

logtools/resource/tests/cassettes/test_elasticsearch_log_repo/test_should_retrieve_failing_test_runs_over_a_single_run_id.yaml

@@ -1,480 +0,0 @@
-interactions:
-- request:
-    body: '{"query":{"bool":{"filter":[{"term":{"runid.keyword":"20240206-093136"}},{"term":{"status.keyword":"Failed"}}]}},"size":10000,"sort":[{"@timestamp":"desc"}]}'
-    headers:
-      accept:
-      - application/vnd.elasticsearch+json; compatible-with=8
-      connection:
-      - keep-alive
-      content-type:
-      - application/vnd.elasticsearch+json; compatible-with=8
-      user-agent:
-      - elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0)
-      x-elastic-client-meta:
-      - es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7
-    method: POST
-    uri: http://localhost:9200/continuous-tests-status-*/_search
-  response:
-    body:
-      string: '{"took":5,"timed_out":false,"_shards":{"total":24,"successful":24,"skipped":0,"failed":0},"hits":{"total":{"value":1,"relation":"eq"},"max_score":null,"hits":[{"_index":"continuous-tests-status-2024.02.07","_id":"t5apg40BTe2pYqi1l9b8","_score":null,"_ignored":["message.keyword"],"_source":{"@timestamp":"2024-02-06T23:07:39.6615501Z","avgupload":"297.8KB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
-        build","codexrevision":"6869475","error":" data/zDvZRwzm5UemKDPMvadCu999HrqUnCJGzvKnsF7eiy2XV3TzoW7V/network''
-        timed out after 3 tries over 10 mins, 1 secs.","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":"<CODEX24>,<BOOTSTRAP2>","message":"System.Exception:
-        System.TimeoutException: Retry ''HTTP-GET-STREAM: data/zDvZRwzm5UemKDPMvadCu999HrqUnCJGzvKnsF7eiy2XV3TzoW7V/network''
-        timed out after 3 tries over 10 mins, 1 secs.\n ---> System.AggregateException:
-        One or more errors occurred. (One or more errors occurred. (A task was canceled.))
-        (One or more errors occurred. (A task was canceled.)) (One or more errors
-        occurred. (A task was canceled.)) (One or more errors occurred. (Connection
-        refused (bootstrap-2-2-int.two-client-test-bugfix.svc.cluster.local:30014)))\n
-        ---> System.AggregateException: One or more errors occurred. (A task was canceled.)\n
-        ---> System.Threading.Tasks.TaskCanceledException: A task was canceled.\n   at
-        System.Threading.Tasks.Task.GetExceptions(Boolean includeTaskCanceledExceptions)\n   at
-        System.Threading.Tasks.Task.ThrowIfExceptional(Boolean includeTaskCanceledExceptions)\n   at
-        System.Threading.Tasks.Task.Wait(Int32 millisecondsTimeout, CancellationToken
-        cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at Utils.Time.Wait[T](Task`1
-        task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0()
-        in /cs-codex-dist-tests/Framework/Core/Http.cs:line 125\n   at Utils.Time.Retry[T](Func`1
-        action, Int32 maxRetries, TimeSpan retryTime, String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
-        144\n   at Core.Http.LockRetry[T](Func`1 operation, String description) in
-        /cs-codex-dist-tests/Framework/Core/Http.cs:line 212\n   at Core.Http.HttpGetStream(String
-        route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line 120\n   at CodexPlugin.CodexAccess.DownloadFile(String
-        contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line
-        71\n   at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile
-        file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
-        200\n   at CodexPlugin.CodexNode.<>c__DisplayClass35_0.<DownloadContent>b__0()
-        in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n   at
-        Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug)
-        in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n   at CodexPlugin.CodexNode.DownloadContent(ContentId
-        contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
-        128\n   at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.<DownloadTestFile>b__0()
-        in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        43\n   at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action
-        action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        82\n   at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        43\n   at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n   at
-        System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
-        invokeAttr)\n   at System.Reflection.RuntimeMethodInfo.Invoke(Object obj,
-        BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n   at
-        ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke)
-        in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n   at
-        ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        250\n   at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        150\n   at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
-        /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n   at
-        ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.<Run>b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        53\n   at Utils.TaskFactory.CatchException(Action action, String name) in
-        /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n   at Utils.TaskFactory.<>c__DisplayClass2_0.<Run>b__0()
-        in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n   at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread
-        threadPoolThread, ExecutionContext executionContext, ContextCallback callback,
-        Object state)\n   at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task&
-        currentTaskSlot, Thread threadPoolThread)\n   at System.Threading.ThreadPoolWorkQueue.Dispatch()\n   at
-        System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n---
-        End of stack trace from previous location ---\n\n   --- End of inner exception
-        stack trace ---\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
-        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
-        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
-        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
-        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
-        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n   ---
-        End of inner exception stack trace ---\n ---> (Inner Exception #1) System.AggregateException:
-        One or more errors occurred. (A task was canceled.)\n ---> System.Threading.Tasks.TaskCanceledException:
-        A task was canceled.\n   at System.Threading.Tasks.Task.GetExceptions(Boolean
-        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
-        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
-        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
-        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
-        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
-        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n   at
-        Core.Http.LockRetry[T](Func`1 operation, String description) in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        212\n   at Core.Http.HttpGetStream(String route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        120\n   at CodexPlugin.CodexAccess.DownloadFile(String contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line
-        71\n   at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile
-        file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
-        200\n   at CodexPlugin.CodexNode.<>c__DisplayClass35_0.<DownloadContent>b__0()
-        in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n   at
-        Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug)
-        in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n   at CodexPlugin.CodexNode.DownloadContent(ContentId
-        contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
-        128\n   at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.<DownloadTestFile>b__0()
-        in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        43\n   at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action
-        action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        82\n   at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        43\n   at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n   at
-        System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
-        invokeAttr)\n   at System.Reflection.RuntimeMethodInfo.Invoke(Object obj,
-        BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n   at
-        ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke)
-        in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n   at
-        ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        250\n   at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        150\n   at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
-        /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n   at
-        ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.<Run>b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        53\n   at Utils.TaskFactory.CatchException(Action action, String name) in
-        /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n   at Utils.TaskFactory.<>c__DisplayClass2_0.<Run>b__0()
-        in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n   at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread
-        threadPoolThread, ExecutionContext executionContext, ContextCallback callback,
-        Object state)\n   at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task&
-        currentTaskSlot, Thread threadPoolThread)\n   at System.Threading.ThreadPoolWorkQueue.Dispatch()\n   at
-        System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n---
-        End of stack trace from previous location ---\n\n   --- End of inner exception
-        stack trace ---\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
-        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
-        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
-        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
-        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
-        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n
-        ---> (Inner Exception #2) System.AggregateException: One or more errors occurred.
-        (A task was canceled.)\n ---> System.Threading.Tasks.TaskCanceledException:
-        A task was canceled.\n   at System.Threading.Tasks.Task.GetExceptions(Boolean
-        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
-        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
-        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
-        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
-        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
-        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n   at
-        Core.Http.LockRetry[T](Func`1 operation, String description) in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        212\n   at Core.Http.HttpGetStream(String route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        120\n   at CodexPlugin.CodexAccess.DownloadFile(String contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line
-        71\n   at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile
-        file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
-        200\n   at CodexPlugin.CodexNode.<>c__DisplayClass35_0.<DownloadContent>b__0()
-        in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n   at
-        Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug)
-        in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n   at CodexPlugin.CodexNode.DownloadContent(ContentId
-        contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
-        128\n   at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.<DownloadTestFile>b__0()
-        in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        43\n   at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action
-        action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        82\n   at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        43\n   at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n   at
-        System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
-        invokeAttr)\n   at System.Reflection.RuntimeMethodInfo.Invoke(Object obj,
-        BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n   at
-        ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke)
-        in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n   at
-        ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        250\n   at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        150\n   at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
-        /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n   at
-        ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.<Run>b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        53\n   at Utils.TaskFactory.CatchException(Action action, String name) in
-        /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n   at Utils.TaskFactory.<>c__DisplayClass2_0.<Run>b__0()
-        in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n   at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread
-        threadPoolThread, ExecutionContext executionContext, ContextCallback callback,
-        Object state)\n   at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task&
-        currentTaskSlot, Thread threadPoolThread)\n   at System.Threading.ThreadPoolWorkQueue.Dispatch()\n   at
-        System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n---
-        End of stack trace from previous location ---\n\n   --- End of inner exception
-        stack trace ---\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
-        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
-        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
-        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
-        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
-        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n
-        ---> (Inner Exception #3) System.AggregateException: One or more errors occurred.
-        (Connection refused (bootstrap-2-2-int.two-client-test-bugfix.svc.cluster.local:30014))\n
-        ---> System.Net.Http.HttpRequestException: Connection refused (bootstrap-2-2-int.two-client-test-bugfix.svc.cluster.local:30014)\n
-        ---> System.Net.Sockets.SocketException (111): Connection refused\n   at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.ThrowException(SocketError
-        error, CancellationToken cancellationToken)\n   at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.System.Threading.Tasks.Sources.IValueTaskSource.GetResult(Int16
-        token)\n   at System.Net.Sockets.Socket.<ConnectAsync>g__WaitForConnectWithCancellation|281_0(AwaitableSocketAsyncEventArgs
-        saea, ValueTask connectTask, CancellationToken cancellationToken)\n   at System.Net.Http.HttpConnectionPool.ConnectToTcpHostAsync(String
-        host, Int32 port, HttpRequestMessage initialRequest, Boolean async, CancellationToken
-        cancellationToken)\n   --- End of inner exception stack trace ---\n   at System.Net.Http.HttpConnectionPool.ConnectToTcpHostAsync(String
-        host, Int32 port, HttpRequestMessage initialRequest, Boolean async, CancellationToken
-        cancellationToken)\n   at System.Net.Http.HttpConnectionPool.ConnectAsync(HttpRequestMessage
-        request, Boolean async, CancellationToken cancellationToken)\n   at System.Net.Http.HttpConnectionPool.CreateHttp11ConnectionAsync(HttpRequestMessage
-        request, Boolean async, CancellationToken cancellationToken)\n   at System.Net.Http.HttpConnectionPool.AddHttp11ConnectionAsync(QueueItem
-        queueItem)\n   at System.Threading.Tasks.TaskCompletionSourceWithCancellation`1.WaitWithCancellationAsync(CancellationToken
-        cancellationToken)\n   at System.Net.Http.HttpConnectionPool.HttpConnectionWaiter`1.WaitForConnectionAsync(Boolean
-        async, CancellationToken requestCancellationToken)\n   at System.Net.Http.HttpConnectionPool.SendWithVersionDetectionAndRetryAsync(HttpRequestMessage
-        request, Boolean async, Boolean doRequestAuth, CancellationToken cancellationToken)\n   at
-        System.Net.Http.RedirectHandler.SendAsync(HttpRequestMessage request, Boolean
-        async, CancellationToken cancellationToken)\n   at System.Net.Http.HttpClient.GetStreamAsyncCore(HttpRequestMessage
-        request, CancellationToken cancellationToken)\n   --- End of inner exception
-        stack trace ---\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
-        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
-        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
-        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
-        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
-        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n   ---
-        End of inner exception stack trace ---\n   at Utils.Time.Retry[T](Func`1 action,
-        Int32 maxRetries, TimeSpan retryTime, String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
-        139\n   at Core.Http.LockRetry[T](Func`1 operation, String description) in
-        /cs-codex-dist-tests/Framework/Core/Http.cs:line 212\n   at Core.Http.HttpGetStream(String
-        route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line 120\n   at CodexPlugin.CodexAccess.DownloadFile(String
-        contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line
-        71\n   at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile
-        file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
-        200\n   at CodexPlugin.CodexNode.<>c__DisplayClass35_0.<DownloadContent>b__0()
-        in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n   at
-        Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug)
-        in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n   at CodexPlugin.CodexNode.DownloadContent(ContentId
-        contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
-        128\n   at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.<DownloadTestFile>b__0()
-        in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        43\n   at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action
-        action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        82\n   at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        43\n   at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n   at
-        System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
-        invokeAttr)\n   at ContinuousTests.SingleTestRun.ThrowFailTest() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        186\n   at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        155\n   at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
-        /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Failed","testduration":"0","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T22:52:53.0881883Z","testtype":"continuous-tests"},"sort":[1707260859661]}]}}'
-    headers:
-      Transfer-Encoding:
-      - chunked
-      X-elastic-product:
-      - Elasticsearch
-      content-type:
-      - application/vnd.elasticsearch+json;compatible-with=8
-    status:
-      code: 200
-      message: OK
-- request:
-    body: null
-    headers:
-      accept:
-      - application/vnd.elasticsearch+json; compatible-with=8
-      connection:
-      - keep-alive
-      user-agent:
-      - elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0)
-      x-elastic-client-meta:
-      - es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7
-    method: GET
-    uri: http://localhost:9200/continuous-tests-status-2024.02.07/_doc/t5apg40BTe2pYqi1l9b8
-  response:
-    body:
-      string: '{"_index":"continuous-tests-status-2024.02.07","_id":"t5apg40BTe2pYqi1l9b8","_version":1,"_seq_no":211,"_primary_term":1,"found":true,"_source":{"@timestamp":"2024-02-06T23:07:39.6615501Z","avgupload":"297.8KB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
-        build","codexrevision":"6869475","error":" data/zDvZRwzm5UemKDPMvadCu999HrqUnCJGzvKnsF7eiy2XV3TzoW7V/network''
-        timed out after 3 tries over 10 mins, 1 secs.","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":"<CODEX24>,<BOOTSTRAP2>","message":"System.Exception:
-        System.TimeoutException: Retry ''HTTP-GET-STREAM: data/zDvZRwzm5UemKDPMvadCu999HrqUnCJGzvKnsF7eiy2XV3TzoW7V/network''
-        timed out after 3 tries over 10 mins, 1 secs.\n ---> System.AggregateException:
-        One or more errors occurred. (One or more errors occurred. (A task was canceled.))
-        (One or more errors occurred. (A task was canceled.)) (One or more errors
-        occurred. (A task was canceled.)) (One or more errors occurred. (Connection
-        refused (bootstrap-2-2-int.two-client-test-bugfix.svc.cluster.local:30014)))\n
-        ---> System.AggregateException: One or more errors occurred. (A task was canceled.)\n
-        ---> System.Threading.Tasks.TaskCanceledException: A task was canceled.\n   at
-        System.Threading.Tasks.Task.GetExceptions(Boolean includeTaskCanceledExceptions)\n   at
-        System.Threading.Tasks.Task.ThrowIfExceptional(Boolean includeTaskCanceledExceptions)\n   at
-        System.Threading.Tasks.Task.Wait(Int32 millisecondsTimeout, CancellationToken
-        cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at Utils.Time.Wait[T](Task`1
-        task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0()
-        in /cs-codex-dist-tests/Framework/Core/Http.cs:line 125\n   at Utils.Time.Retry[T](Func`1
-        action, Int32 maxRetries, TimeSpan retryTime, String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
-        144\n   at Core.Http.LockRetry[T](Func`1 operation, String description) in
-        /cs-codex-dist-tests/Framework/Core/Http.cs:line 212\n   at Core.Http.HttpGetStream(String
-        route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line 120\n   at CodexPlugin.CodexAccess.DownloadFile(String
-        contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line
-        71\n   at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile
-        file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
-        200\n   at CodexPlugin.CodexNode.<>c__DisplayClass35_0.<DownloadContent>b__0()
-        in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n   at
-        Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug)
-        in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n   at CodexPlugin.CodexNode.DownloadContent(ContentId
-        contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
-        128\n   at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.<DownloadTestFile>b__0()
-        in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        43\n   at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action
-        action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        82\n   at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        43\n   at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n   at
-        System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
-        invokeAttr)\n   at System.Reflection.RuntimeMethodInfo.Invoke(Object obj,
-        BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n   at
-        ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke)
-        in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n   at
-        ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        250\n   at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        150\n   at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
-        /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n   at
-        ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.<Run>b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        53\n   at Utils.TaskFactory.CatchException(Action action, String name) in
-        /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n   at Utils.TaskFactory.<>c__DisplayClass2_0.<Run>b__0()
-        in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n   at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread
-        threadPoolThread, ExecutionContext executionContext, ContextCallback callback,
-        Object state)\n   at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task&
-        currentTaskSlot, Thread threadPoolThread)\n   at System.Threading.ThreadPoolWorkQueue.Dispatch()\n   at
-        System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n---
-        End of stack trace from previous location ---\n\n   --- End of inner exception
-        stack trace ---\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
-        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
-        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
-        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
-        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
-        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n   ---
-        End of inner exception stack trace ---\n ---> (Inner Exception #1) System.AggregateException:
-        One or more errors occurred. (A task was canceled.)\n ---> System.Threading.Tasks.TaskCanceledException:
-        A task was canceled.\n   at System.Threading.Tasks.Task.GetExceptions(Boolean
-        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
-        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
-        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
-        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
-        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
-        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n   at
-        Core.Http.LockRetry[T](Func`1 operation, String description) in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        212\n   at Core.Http.HttpGetStream(String route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        120\n   at CodexPlugin.CodexAccess.DownloadFile(String contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line
-        71\n   at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile
-        file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
-        200\n   at CodexPlugin.CodexNode.<>c__DisplayClass35_0.<DownloadContent>b__0()
-        in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n   at
-        Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug)
-        in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n   at CodexPlugin.CodexNode.DownloadContent(ContentId
-        contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
-        128\n   at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.<DownloadTestFile>b__0()
-        in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        43\n   at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action
-        action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        82\n   at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        43\n   at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n   at
-        System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
-        invokeAttr)\n   at System.Reflection.RuntimeMethodInfo.Invoke(Object obj,
-        BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n   at
-        ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke)
-        in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n   at
-        ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        250\n   at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        150\n   at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
-        /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n   at
-        ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.<Run>b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        53\n   at Utils.TaskFactory.CatchException(Action action, String name) in
-        /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n   at Utils.TaskFactory.<>c__DisplayClass2_0.<Run>b__0()
-        in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n   at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread
-        threadPoolThread, ExecutionContext executionContext, ContextCallback callback,
-        Object state)\n   at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task&
-        currentTaskSlot, Thread threadPoolThread)\n   at System.Threading.ThreadPoolWorkQueue.Dispatch()\n   at
-        System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n---
-        End of stack trace from previous location ---\n\n   --- End of inner exception
-        stack trace ---\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
-        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
-        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
-        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
-        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
-        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n
-        ---> (Inner Exception #2) System.AggregateException: One or more errors occurred.
-        (A task was canceled.)\n ---> System.Threading.Tasks.TaskCanceledException:
-        A task was canceled.\n   at System.Threading.Tasks.Task.GetExceptions(Boolean
-        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
-        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
-        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
-        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
-        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
-        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n   at
-        Core.Http.LockRetry[T](Func`1 operation, String description) in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        212\n   at Core.Http.HttpGetStream(String route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        120\n   at CodexPlugin.CodexAccess.DownloadFile(String contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line
-        71\n   at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile
-        file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
-        200\n   at CodexPlugin.CodexNode.<>c__DisplayClass35_0.<DownloadContent>b__0()
-        in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n   at
-        Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug)
-        in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n   at CodexPlugin.CodexNode.DownloadContent(ContentId
-        contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
-        128\n   at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.<DownloadTestFile>b__0()
-        in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        43\n   at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action
-        action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        82\n   at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        43\n   at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n   at
-        System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
-        invokeAttr)\n   at System.Reflection.RuntimeMethodInfo.Invoke(Object obj,
-        BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n   at
-        ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke)
-        in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n   at
-        ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        250\n   at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        150\n   at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
-        /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n   at
-        ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.<Run>b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        53\n   at Utils.TaskFactory.CatchException(Action action, String name) in
-        /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n   at Utils.TaskFactory.<>c__DisplayClass2_0.<Run>b__0()
-        in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n   at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread
-        threadPoolThread, ExecutionContext executionContext, ContextCallback callback,
-        Object state)\n   at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task&
-        currentTaskSlot, Thread threadPoolThread)\n   at System.Threading.ThreadPoolWorkQueue.Dispatch()\n   at
-        System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n---
-        End of stack trace from previous location ---\n\n   --- End of inner exception
-        stack trace ---\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
-        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
-        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
-        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
-        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
-        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n
-        ---> (Inner Exception #3) System.AggregateException: One or more errors occurred.
-        (Connection refused (bootstrap-2-2-int.two-client-test-bugfix.svc.cluster.local:30014))\n
-        ---> System.Net.Http.HttpRequestException: Connection refused (bootstrap-2-2-int.two-client-test-bugfix.svc.cluster.local:30014)\n
-        ---> System.Net.Sockets.SocketException (111): Connection refused\n   at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.ThrowException(SocketError
-        error, CancellationToken cancellationToken)\n   at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.System.Threading.Tasks.Sources.IValueTaskSource.GetResult(Int16
-        token)\n   at System.Net.Sockets.Socket.<ConnectAsync>g__WaitForConnectWithCancellation|281_0(AwaitableSocketAsyncEventArgs
-        saea, ValueTask connectTask, CancellationToken cancellationToken)\n   at System.Net.Http.HttpConnectionPool.ConnectToTcpHostAsync(String
-        host, Int32 port, HttpRequestMessage initialRequest, Boolean async, CancellationToken
-        cancellationToken)\n   --- End of inner exception stack trace ---\n   at System.Net.Http.HttpConnectionPool.ConnectToTcpHostAsync(String
-        host, Int32 port, HttpRequestMessage initialRequest, Boolean async, CancellationToken
-        cancellationToken)\n   at System.Net.Http.HttpConnectionPool.ConnectAsync(HttpRequestMessage
-        request, Boolean async, CancellationToken cancellationToken)\n   at System.Net.Http.HttpConnectionPool.CreateHttp11ConnectionAsync(HttpRequestMessage
-        request, Boolean async, CancellationToken cancellationToken)\n   at System.Net.Http.HttpConnectionPool.AddHttp11ConnectionAsync(QueueItem
-        queueItem)\n   at System.Threading.Tasks.TaskCompletionSourceWithCancellation`1.WaitWithCancellationAsync(CancellationToken
-        cancellationToken)\n   at System.Net.Http.HttpConnectionPool.HttpConnectionWaiter`1.WaitForConnectionAsync(Boolean
-        async, CancellationToken requestCancellationToken)\n   at System.Net.Http.HttpConnectionPool.SendWithVersionDetectionAndRetryAsync(HttpRequestMessage
-        request, Boolean async, Boolean doRequestAuth, CancellationToken cancellationToken)\n   at
-        System.Net.Http.RedirectHandler.SendAsync(HttpRequestMessage request, Boolean
-        async, CancellationToken cancellationToken)\n   at System.Net.Http.HttpClient.GetStreamAsyncCore(HttpRequestMessage
-        request, CancellationToken cancellationToken)\n   --- End of inner exception
-        stack trace ---\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
-        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
-        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
-        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
-        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
-        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n   ---
-        End of inner exception stack trace ---\n   at Utils.Time.Retry[T](Func`1 action,
-        Int32 maxRetries, TimeSpan retryTime, String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
-        139\n   at Core.Http.LockRetry[T](Func`1 operation, String description) in
-        /cs-codex-dist-tests/Framework/Core/Http.cs:line 212\n   at Core.Http.HttpGetStream(String
-        route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line 120\n   at CodexPlugin.CodexAccess.DownloadFile(String
-        contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line
-        71\n   at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile
-        file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
-        200\n   at CodexPlugin.CodexNode.<>c__DisplayClass35_0.<DownloadContent>b__0()
-        in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n   at
-        Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug)
-        in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n   at CodexPlugin.CodexNode.DownloadContent(ContentId
-        contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
-        128\n   at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.<DownloadTestFile>b__0()
-        in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        43\n   at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action
-        action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        82\n   at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        43\n   at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n   at
-        System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
-        invokeAttr)\n   at ContinuousTests.SingleTestRun.ThrowFailTest() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        186\n   at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        155\n   at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
-        /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Failed","testduration":"0","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T22:52:53.0881883Z","testtype":"continuous-tests"}}'
-    headers:
-      X-elastic-product:
-      - Elasticsearch
-      content-length:
-      - '20829'
-      content-type:
-      - application/vnd.elasticsearch+json;compatible-with=8
-    status:
-      code: 200
-      message: OK
-version: 1

logtools/resource/tests/cassettes/test_elasticsearch_log_repo/test_should_retrieve_test_runs_over_a_single_run_id.yaml

@@ -1,255 +0,0 @@
-interactions:
-- request:
-    body: '{"query":{"bool":{"filter":[{"term":{"runid.keyword":"20240206-093136"}}]}},"size":10000,"sort":[{"@timestamp":"desc"}]}'
-    headers:
-      accept:
-      - application/vnd.elasticsearch+json; compatible-with=8
-      connection:
-      - keep-alive
-      content-type:
-      - application/vnd.elasticsearch+json; compatible-with=8
-      user-agent:
-      - elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0)
-      x-elastic-client-meta:
-      - es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7
-    method: POST
-    uri: http://localhost:9200/continuous-tests-status-*/_search
-  response:
-    body:
-      string: '{"took":4,"timed_out":false,"_shards":{"total":24,"successful":24,"skipped":0,"failed":0},"hits":{"total":{"value":14,"relation":"eq"},"max_score":null,"hits":[{"_index":"continuous-tests-status-2024.02.07","_id":"t5apg40BTe2pYqi1l9b8","_score":null,"_ignored":["message.keyword"],"_source":{"@timestamp":"2024-02-06T23:07:39.6615501Z","avgupload":"297.8KB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
-        build","codexrevision":"6869475","error":" data/zDvZRwzm5UemKDPMvadCu999HrqUnCJGzvKnsF7eiy2XV3TzoW7V/network''
-        timed out after 3 tries over 10 mins, 1 secs.","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":"<CODEX24>,<BOOTSTRAP2>","message":"System.Exception:
-        System.TimeoutException: Retry ''HTTP-GET-STREAM: data/zDvZRwzm5UemKDPMvadCu999HrqUnCJGzvKnsF7eiy2XV3TzoW7V/network''
-        timed out after 3 tries over 10 mins, 1 secs.\n ---> System.AggregateException:
-        One or more errors occurred. (One or more errors occurred. (A task was canceled.))
-        (One or more errors occurred. (A task was canceled.)) (One or more errors
-        occurred. (A task was canceled.)) (One or more errors occurred. (Connection
-        refused (bootstrap-2-2-int.two-client-test-bugfix.svc.cluster.local:30014)))\n
-        ---> System.AggregateException: One or more errors occurred. (A task was canceled.)\n
-        ---> System.Threading.Tasks.TaskCanceledException: A task was canceled.\n   at
-        System.Threading.Tasks.Task.GetExceptions(Boolean includeTaskCanceledExceptions)\n   at
-        System.Threading.Tasks.Task.ThrowIfExceptional(Boolean includeTaskCanceledExceptions)\n   at
-        System.Threading.Tasks.Task.Wait(Int32 millisecondsTimeout, CancellationToken
-        cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at Utils.Time.Wait[T](Task`1
-        task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0()
-        in /cs-codex-dist-tests/Framework/Core/Http.cs:line 125\n   at Utils.Time.Retry[T](Func`1
-        action, Int32 maxRetries, TimeSpan retryTime, String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
-        144\n   at Core.Http.LockRetry[T](Func`1 operation, String description) in
-        /cs-codex-dist-tests/Framework/Core/Http.cs:line 212\n   at Core.Http.HttpGetStream(String
-        route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line 120\n   at CodexPlugin.CodexAccess.DownloadFile(String
-        contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line
-        71\n   at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile
-        file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
-        200\n   at CodexPlugin.CodexNode.<>c__DisplayClass35_0.<DownloadContent>b__0()
-        in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n   at
-        Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug)
-        in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n   at CodexPlugin.CodexNode.DownloadContent(ContentId
-        contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
-        128\n   at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.<DownloadTestFile>b__0()
-        in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        43\n   at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action
-        action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        82\n   at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        43\n   at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n   at
-        System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
-        invokeAttr)\n   at System.Reflection.RuntimeMethodInfo.Invoke(Object obj,
-        BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n   at
-        ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke)
-        in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n   at
-        ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        250\n   at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        150\n   at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
-        /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n   at
-        ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.<Run>b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        53\n   at Utils.TaskFactory.CatchException(Action action, String name) in
-        /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n   at Utils.TaskFactory.<>c__DisplayClass2_0.<Run>b__0()
-        in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n   at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread
-        threadPoolThread, ExecutionContext executionContext, ContextCallback callback,
-        Object state)\n   at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task&
-        currentTaskSlot, Thread threadPoolThread)\n   at System.Threading.ThreadPoolWorkQueue.Dispatch()\n   at
-        System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n---
-        End of stack trace from previous location ---\n\n   --- End of inner exception
-        stack trace ---\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
-        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
-        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
-        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
-        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
-        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n   ---
-        End of inner exception stack trace ---\n ---> (Inner Exception #1) System.AggregateException:
-        One or more errors occurred. (A task was canceled.)\n ---> System.Threading.Tasks.TaskCanceledException:
-        A task was canceled.\n   at System.Threading.Tasks.Task.GetExceptions(Boolean
-        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
-        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
-        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
-        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
-        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
-        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n   at
-        Core.Http.LockRetry[T](Func`1 operation, String description) in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        212\n   at Core.Http.HttpGetStream(String route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        120\n   at CodexPlugin.CodexAccess.DownloadFile(String contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line
-        71\n   at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile
-        file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
-        200\n   at CodexPlugin.CodexNode.<>c__DisplayClass35_0.<DownloadContent>b__0()
-        in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n   at
-        Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug)
-        in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n   at CodexPlugin.CodexNode.DownloadContent(ContentId
-        contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
-        128\n   at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.<DownloadTestFile>b__0()
-        in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        43\n   at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action
-        action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        82\n   at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        43\n   at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n   at
-        System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
-        invokeAttr)\n   at System.Reflection.RuntimeMethodInfo.Invoke(Object obj,
-        BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n   at
-        ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke)
-        in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n   at
-        ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        250\n   at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        150\n   at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
-        /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n   at
-        ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.<Run>b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        53\n   at Utils.TaskFactory.CatchException(Action action, String name) in
-        /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n   at Utils.TaskFactory.<>c__DisplayClass2_0.<Run>b__0()
-        in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n   at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread
-        threadPoolThread, ExecutionContext executionContext, ContextCallback callback,
-        Object state)\n   at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task&
-        currentTaskSlot, Thread threadPoolThread)\n   at System.Threading.ThreadPoolWorkQueue.Dispatch()\n   at
-        System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n---
-        End of stack trace from previous location ---\n\n   --- End of inner exception
-        stack trace ---\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
-        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
-        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
-        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
-        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
-        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n
-        ---> (Inner Exception #2) System.AggregateException: One or more errors occurred.
-        (A task was canceled.)\n ---> System.Threading.Tasks.TaskCanceledException:
-        A task was canceled.\n   at System.Threading.Tasks.Task.GetExceptions(Boolean
-        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
-        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
-        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
-        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
-        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
-        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n   at
-        Core.Http.LockRetry[T](Func`1 operation, String description) in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        212\n   at Core.Http.HttpGetStream(String route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        120\n   at CodexPlugin.CodexAccess.DownloadFile(String contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line
-        71\n   at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile
-        file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
-        200\n   at CodexPlugin.CodexNode.<>c__DisplayClass35_0.<DownloadContent>b__0()
-        in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n   at
-        Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug)
-        in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n   at CodexPlugin.CodexNode.DownloadContent(ContentId
-        contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
-        128\n   at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.<DownloadTestFile>b__0()
-        in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        43\n   at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action
-        action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        82\n   at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        43\n   at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n   at
-        System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
-        invokeAttr)\n   at System.Reflection.RuntimeMethodInfo.Invoke(Object obj,
-        BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n   at
-        ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke)
-        in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n   at
-        ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        250\n   at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        150\n   at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
-        /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n   at
-        ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.<Run>b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        53\n   at Utils.TaskFactory.CatchException(Action action, String name) in
-        /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n   at Utils.TaskFactory.<>c__DisplayClass2_0.<Run>b__0()
-        in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n   at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread
-        threadPoolThread, ExecutionContext executionContext, ContextCallback callback,
-        Object state)\n   at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task&
-        currentTaskSlot, Thread threadPoolThread)\n   at System.Threading.ThreadPoolWorkQueue.Dispatch()\n   at
-        System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n---
-        End of stack trace from previous location ---\n\n   --- End of inner exception
-        stack trace ---\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
-        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
-        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
-        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
-        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
-        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n
-        ---> (Inner Exception #3) System.AggregateException: One or more errors occurred.
-        (Connection refused (bootstrap-2-2-int.two-client-test-bugfix.svc.cluster.local:30014))\n
-        ---> System.Net.Http.HttpRequestException: Connection refused (bootstrap-2-2-int.two-client-test-bugfix.svc.cluster.local:30014)\n
-        ---> System.Net.Sockets.SocketException (111): Connection refused\n   at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.ThrowException(SocketError
-        error, CancellationToken cancellationToken)\n   at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.System.Threading.Tasks.Sources.IValueTaskSource.GetResult(Int16
-        token)\n   at System.Net.Sockets.Socket.<ConnectAsync>g__WaitForConnectWithCancellation|281_0(AwaitableSocketAsyncEventArgs
-        saea, ValueTask connectTask, CancellationToken cancellationToken)\n   at System.Net.Http.HttpConnectionPool.ConnectToTcpHostAsync(String
-        host, Int32 port, HttpRequestMessage initialRequest, Boolean async, CancellationToken
-        cancellationToken)\n   --- End of inner exception stack trace ---\n   at System.Net.Http.HttpConnectionPool.ConnectToTcpHostAsync(String
-        host, Int32 port, HttpRequestMessage initialRequest, Boolean async, CancellationToken
-        cancellationToken)\n   at System.Net.Http.HttpConnectionPool.ConnectAsync(HttpRequestMessage
-        request, Boolean async, CancellationToken cancellationToken)\n   at System.Net.Http.HttpConnectionPool.CreateHttp11ConnectionAsync(HttpRequestMessage
-        request, Boolean async, CancellationToken cancellationToken)\n   at System.Net.Http.HttpConnectionPool.AddHttp11ConnectionAsync(QueueItem
-        queueItem)\n   at System.Threading.Tasks.TaskCompletionSourceWithCancellation`1.WaitWithCancellationAsync(CancellationToken
-        cancellationToken)\n   at System.Net.Http.HttpConnectionPool.HttpConnectionWaiter`1.WaitForConnectionAsync(Boolean
-        async, CancellationToken requestCancellationToken)\n   at System.Net.Http.HttpConnectionPool.SendWithVersionDetectionAndRetryAsync(HttpRequestMessage
-        request, Boolean async, Boolean doRequestAuth, CancellationToken cancellationToken)\n   at
-        System.Net.Http.RedirectHandler.SendAsync(HttpRequestMessage request, Boolean
-        async, CancellationToken cancellationToken)\n   at System.Net.Http.HttpClient.GetStreamAsyncCore(HttpRequestMessage
-        request, CancellationToken cancellationToken)\n   --- End of inner exception
-        stack trace ---\n   at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
-        includeTaskCanceledExceptions)\n   at System.Threading.Tasks.Task.Wait(Int32
-        millisecondsTimeout, CancellationToken cancellationToken)\n   at System.Threading.Tasks.Task.Wait()\n   at
-        Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
-        12\n   at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
-        125\n   at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
-        String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n   ---
-        End of inner exception stack trace ---\n   at Utils.Time.Retry[T](Func`1 action,
-        Int32 maxRetries, TimeSpan retryTime, String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
-        139\n   at Core.Http.LockRetry[T](Func`1 operation, String description) in
-        /cs-codex-dist-tests/Framework/Core/Http.cs:line 212\n   at Core.Http.HttpGetStream(String
-        route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line 120\n   at CodexPlugin.CodexAccess.DownloadFile(String
-        contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line
-        71\n   at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile
-        file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
-        200\n   at CodexPlugin.CodexNode.<>c__DisplayClass35_0.<DownloadContent>b__0()
-        in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n   at
-        Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug)
-        in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n   at CodexPlugin.CodexNode.DownloadContent(ContentId
-        contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
-        128\n   at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.<DownloadTestFile>b__0()
-        in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        43\n   at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action
-        action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        82\n   at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
-        43\n   at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n   at
-        System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
-        invokeAttr)\n   at ContinuousTests.SingleTestRun.ThrowFailTest() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        186\n   at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
-        155\n   at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
-        /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Failed","testduration":"0","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T22:52:53.0881883Z","testtype":"continuous-tests"},"sort":[1707260859661]},{"_index":"continuous-tests-status-2024.02.07","_id":"tpapg40BTe2pYqi1l9b8","_score":null,"_source":{"@timestamp":"2024-02-06T22:52:52.2996375Z","avgdownload":"780.8KB/s","avgupload":"3MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
-        build","codexrevision":"6869475","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":"<BOOTSTRAP2>,<CODEX13>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Passed","testduration":"141.9114191","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T22:50:30.3878431Z","testtype":"continuous-tests"},"sort":[1707259972299]},{"_index":"continuous-tests-status-2024.02.07","_id":"sZapg40BTe2pYqi1l9b8","_score":null,"_source":{"@timestamp":"2024-02-06T22:50:30.0675681Z","avgdownload":"152.9KB/s","avgupload":"3.1MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
-        build","codexrevision":"6869475","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":"<CODEX24>,<CODEX13>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Passed","testduration":"572.9451525","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T22:40:57.1222113Z","testtype":"continuous-tests"},"sort":[1707259830067]},{"_index":"continuous-tests-status-2024.02.07","_id":"sJapg40BTe2pYqi1l9b8","_score":null,"_source":{"@timestamp":"2024-02-06T22:40:56.1120614Z","avgdownload":"70KB/s","avgupload":"3.1MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
-        build","codexrevision":"6869475","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":"<BOOTSTRAP2>,<CODEX24>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Passed","testduration":"1207.5289232","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T22:20:48.5829991Z","testtype":"continuous-tests"},"sort":[1707259256112]},{"_index":"continuous-tests-status-2024.02.07","_id":"r5apg40BTe2pYqi1l9b8","_score":null,"_source":{"@timestamp":"2024-02-06T22:20:47.5015704Z","avgdownload":"107KB/s","avgupload":"2.9MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
-        build","codexrevision":"6869475","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":"<CODEX13>,<CODEX24>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Passed","testduration":"804.6780181","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T22:07:22.8233627Z","testtype":"continuous-tests"},"sort":[1707258047501]},{"_index":"continuous-tests-status-2024.02.07","_id":"rpapg40BTe2pYqi1l9b8","_score":null,"_source":{"@timestamp":"2024-02-06T22:07:13.1890259Z","avgdownload":"1.2MB/s","avgupload":"2.5MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
-        build","codexrevision":"6869475","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":"<CODEX24>,<BOOTSTRAP2>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Passed","testduration":"113.1299709","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T22:05:20.0552568Z","testtype":"continuous-tests"},"sort":[1707257233189]},{"_index":"continuous-tests-status-2024.02.07","_id":"qJapg40BTe2pYqi1l9b8","_score":null,"_source":{"@timestamp":"2024-02-06T22:05:19.1233015Z","avgdownload":"128.4KB/s","avgupload":"2.3MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
-        build","codexrevision":"6869475","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":"<CODEX13>,<CODEX24>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Passed","testduration":"683.6157221","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T21:53:55.5063178Z","testtype":"continuous-tests"},"sort":[1707257119123]},{"_index":"continuous-tests-status-2024.02.07","_id":"p5apg40BTe2pYqi1l9b8","_score":null,"_source":{"@timestamp":"2024-02-06T21:53:47.1782269Z","avgdownload":"1MB/s","avgupload":"3.4MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
-        build","codexrevision":"6869475","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":"<BOOTSTRAP2>,<CODEX13>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Passed","testduration":"113.2005889","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T21:51:53.9774688Z","testtype":"continuous-tests"},"sort":[1707256427178]},{"_index":"continuous-tests-status-2024.02.07","_id":"ppapg40BTe2pYqi1l9b8","_score":null,"_source":{"@timestamp":"2024-02-06T21:51:45.5038683Z","avgdownload":"988.9KB/s","avgupload":"4.2MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
-        build","codexrevision":"6869475","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":"<BOOTSTRAP2>,<CODEX13>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Passed","testduration":"112.4111551","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T21:49:53.0925544Z","testtype":"continuous-tests"},"sort":[1707256305503]},{"_index":"continuous-tests-status-2024.02.07","_id":"pZapg40BTe2pYqi1l9b8","_score":null,"_source":{"@timestamp":"2024-02-06T21:49:11.8557349Z","avgdownload":"1.8MB/s","avgupload":"3.3MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
-        build","codexrevision":"6869475","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":"<CODEX24>,<BOOTSTRAP2>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Passed","testduration":"78.958915","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T21:47:52.8966341Z","testtype":"continuous-tests"},"sort":[1707256151855]},{"_index":"continuous-tests-status-2024.02.07","_id":"mpapg40BTe2pYqi1l9b8","_score":null,"_source":{"@timestamp":"2024-02-06T21:47:33.8366307Z","avgdownload":"1.1MB/s","avgupload":"4.3MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
-        build","codexrevision":"6869475","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":"<BOOTSTRAP2>,<CODEX24>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Passed","testduration":"101.1595982","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T21:45:52.6769016Z","testtype":"continuous-tests"},"sort":[1707256053836]},{"_index":"continuous-tests-status-2024.02.07","_id":"mZapg40BTe2pYqi1l9b8","_score":null,"_source":{"@timestamp":"2024-02-06T21:45:08.0812184Z","avgdownload":"1.7MB/s","avgupload":"4.1MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
-        build","codexrevision":"6869475","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":"<CODEX24>,<CODEX13>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Passed","testduration":"75.6624344","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T21:43:52.4185559Z","testtype":"continuous-tests"},"sort":[1707255908081]},{"_index":"continuous-tests-status-2024.02.07","_id":"mJapg40BTe2pYqi1l9b8","_score":null,"_source":{"@timestamp":"2024-02-06T21:43:14.2974430Z","avgdownload":"1.8MB/s","avgupload":"3.1MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
-        build","codexrevision":"6869475","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":"<CODEX24>,<BOOTSTRAP2>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Passed","testduration":"82.0906635","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T21:41:52.2061498Z","testtype":"continuous-tests"},"sort":[1707255794297]},{"_index":"continuous-tests-status-2024.02.07","_id":"l5apg40BTe2pYqi1l9b8","_score":null,"_source":{"@timestamp":"2024-02-06T21:41:05.3802809Z","avgdownload":"1.8MB/s","avgupload":"4.4MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
-        build","codexrevision":"6869475","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":"<CODEX24>,<CODEX13>","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Passed","testduration":"73.3825081","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T21:39:51.9897720Z","testtype":"continuous-tests"},"sort":[1707255665380]}]}}'
-    headers:
-      Transfer-Encoding:
-      - chunked
-      X-elastic-product:
-      - Elasticsearch
-      content-type:
-      - application/vnd.elasticsearch+json;compatible-with=8
-    status:
-      code: 200
-      message: OK
-version: 1

logtools/resource/tests/test_elasticsearch_log_repo.py

@@ -85,7 +85,7 @@ def test_should_respect_time_horizon_for_retrieving_resources():
 @pytest.mark.vcr
 def test_should_retrieve_test_runs_over_a_single_run_id():
     repo = ElasticSearchLogRepo()
-    runs = list(repo.test_runs('20240206-093136'))
+    runs = list(repo.test_runs('20240208-044040'))
 
     assert len(runs) == 14
 
@@ -93,10 +93,10 @@ def test_should_retrieve_test_runs_over_a_single_run_id():
 @pytest.mark.vcr
 def test_should_retrieve_failing_test_runs_over_a_single_run_id():
     repo = ElasticSearchLogRepo()
-    runs = list(repo.test_runs('20240206-093136', failed_only=True))
+    runs = list(repo.test_runs('20240208-044040', failed_only=True))
 
     assert len(runs) == 1
-    assert repo.describe_test_run(runs[0].id).error.strip() == (
+    assert repo.test_run(runs[0].id).error.strip() == (
         "data/zDvZRwzm5UemKDPMvadCu999HrqUnCJGzvKnsF7eiy2XV3TzoW7V/network' timed out after "
         "3 tries over 10 mins, 1 secs."
     )

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "logtools"
-version = "1.1.4"
+version = "1.2.0"
 description = ""
 authors = ["gmega <giuliano@status.im>"]
 readme = "README.md"