From d83bf8fdbb6739b34b4b50b7c1a96185b1b9cded Mon Sep 17 00:00:00 2001 From: gmega Date: Thu, 8 Feb 2024 14:55:56 -0300 Subject: [PATCH] allow retrieval of logs from individual test runs --- logtools/cli/es_logs.py | 170 ++++--- logtools/resource/core.py | 14 +- logtools/resource/elastic_search_log_repo.py | 14 +- ...ailing_test_runs_over_a_single_run_id.yaml | 480 ------------------ ...trieve_test_runs_over_a_single_run_id.yaml | 255 ---------- .../tests/test_elasticsearch_log_repo.py | 6 +- pyproject.toml | 2 +- 7 files changed, 116 insertions(+), 825 deletions(-) delete mode 100644 logtools/resource/tests/cassettes/test_elasticsearch_log_repo/test_should_retrieve_failing_test_runs_over_a_single_run_id.yaml delete mode 100644 logtools/resource/tests/cassettes/test_elasticsearch_log_repo/test_should_retrieve_test_runs_over_a_single_run_id.yaml diff --git a/logtools/cli/es_logs.py b/logtools/cli/es_logs.py index 4d791dd..ff5a4dc 100644 --- a/logtools/cli/es_logs.py +++ b/logtools/cli/es_logs.py @@ -4,9 +4,8 @@ from argparse import ArgumentParser from datetime import timedelta, datetime from enum import Enum from json import JSONEncoder -from typing import List, Iterable, Any +from typing import List, Iterable, Any, Optional, Set -import rich from colored import Style from dateutil import parser as tsparser from elasticsearch import Elasticsearch @@ -26,17 +25,79 @@ class ResourceType(Enum): runs = 'runs' -GETTERS = { +RESOURCE_GETTERS = { ResourceType.pods: lambda repo, args: repo.pods(prefix=args.prefix, run_id=args.run_id), ResourceType.namespaces: lambda repo, args: repo.namespaces(prefix=args.prefix), ResourceType.runs: lambda repo, args: repo.test_runs(run_id=args.run_id, failed_only=args.failed_only), } -DESCRIBERS = { - ResourceType.runs: lambda repo, args: repo.describe_test_run(test_run_id=args.test_run_id), +RESOURCE_DESCRIBERS = { + ResourceType.runs: lambda repo, args: repo.test_run(test_run_id=args.test_run_id), } +def main(): + parser = ArgumentParser() + parser.add_argument( + '--es-host', + help='ElasticSearch URL (defaults to http://localhost:9200)', + default=os.environ.get('ES_HOST', 'http://localhost:9200') + ) + + subparsers = parser.add_subparsers(title='Command', required=True) + _add_get_cli(subparsers) + _add_describe_cli(subparsers) + _add_logs_cli(subparsers) + + args = parser.parse_args() + client = Elasticsearch(args.es_host, request_timeout=60) + args.main(args, client) + + +def get_object(args, client: Elasticsearch): + repo = ElasticSearchLogRepo(client=client) + Console().print(format_table(RESOURCE_GETTERS[ResourceType[args.resource_type]](repo, args))) + + +def describe_object(args, client: Elasticsearch): + repo = ElasticSearchLogRepo(client=client) + Console().print(format_json([RESOURCE_DESCRIBERS[ResourceType[args.resource_type]](repo, args)])) + + +# FIXME this is starting to get too complex to be here. +def get_logs(args, client: Elasticsearch): + resource = ResourceType[args.resource_type] + if resource == ResourceType.pods: + get_pod_logs( + pods=args.pods, + client=client, + colored_output=not args.no_color, + start_date=args.from_, + end_date=args.to, + ) + elif resource == ResourceType.runs: + run = ElasticSearchLogRepo(client=client).test_run(test_run_id=args.test_run_id).test_run + get_pod_logs(set(run.pods), client, start_date=run.start, end_date=run.end) + + +def get_pod_logs(pods: Set[str], + client: Elasticsearch, + colored_output: bool = True, + start_date: Optional[datetime] = None, + end_date: Optional[datetime] = None): + colors = ColorMap() + for line in ElasticSearchSource( + pods=pods, + client=client, + start_date=start_date, + end_date=end_date, + ): + output = f'[{line.location.pod_name}]: {line.raw}' + if colored_output: + output = f'{colors[line.location.pod_name]}{output}{Style.reset}' + print(output) + + def format_table(objects: List, title: str = 'Results') -> Table: tbl = Table(title=title) field_names = None @@ -75,95 +136,60 @@ def _format_field(field: Any): return str(field) -def get_object(args, client: Elasticsearch): - repo = ElasticSearchLogRepo(client=client) - Console().print(format_table(GETTERS[ResourceType[args.resource_type]](repo, args))) - - -def describe_object(args, client: Elasticsearch): - repo = ElasticSearchLogRepo(client=client) - Console().print(format_json([DESCRIBERS[ResourceType[args.resource_type]](repo, args)])) - - -def get_logs(args, client: Elasticsearch): - colors = ColorMap() - for line in ElasticSearchSource( - pods=args.pods, - client=client, - start_date=args.from_, - end_date=args.to, - ): - output = f'[{line.location.pod_name}]: {line.raw}' - if not args.no_color: - output = f'{colors[line.location.pod_name]}{output}{Style.reset}' - print(output) - - -def main(): - parser = ArgumentParser() - parser.add_argument( - '--es-host', - help='ElasticSearch URL (defaults to http://localhost:9200)', - default=os.environ.get('ES_HOST', 'http://localhost:9200') - ) - - subparsers = parser.add_subparsers(title='Command', required=True) - _add_get_cli(subparsers) - _add_describe_cli(subparsers) - _add_logs_cli(subparsers) - - args = parser.parse_args() - client = Elasticsearch(args.es_host, request_timeout=60) - args.main(args, client) - - def _add_get_cli(subparsers): - get = subparsers.add_parser('get', help='Display existing resources') + get = subparsers.add_parser('get', help='display existing resources') get.add_argument('--from', type=tsparser.parse, - help='Show resources present in log messages starting at the given date ' - '(MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm). Defaults to 7 days ago.', + help='show resources present in log messages starting at the given date ' + '(mm-dd-yyyy, or mm-dd-yyyy hh:mm:ss.mmmmmm). defaults to 7 days ago.', default=(datetime.today() - timedelta(days=7)).date()) get.set_defaults(main=get_object) - get_subparsers = get.add_subparsers(title='Resource type', dest='resource_type', required=True) - get_pods = get_subparsers.add_parser('pods', help='Display existing pods') - get_pods.add_argument('--prefix', help='Filter pods by prefix') - get_pods.add_argument('--run-id', help='Show pods for a given run', required=True) + get_subparsers = get.add_subparsers(title='resource type', dest='resource_type', required=True) + get_pods = get_subparsers.add_parser('pods', help='display existing pods') + get_pods.add_argument('--prefix', help='filter pods by prefix') + get_pods.add_argument('--run-id', help='show pods for a given run', required=True) - get_namespaces = get_subparsers.add_parser('namespaces', help='Display existing namespaces') - get_namespaces.add_argument('--prefix', help='Filter namespaces by prefix') + get_namespaces = get_subparsers.add_parser('namespaces', help='display existing namespaces') + get_namespaces.add_argument('--prefix', help='filter namespaces by prefix') - get_namespaces = get_subparsers.add_parser('runs', help='Display current test runs') - get_namespaces.add_argument('--run-id', help='Show test runs for the given run id', required=True) - get_namespaces.add_argument('--failed-only', action='store_true', help='Show only failed test runs') + get_namespaces = get_subparsers.add_parser('runs', help='display current test runs') + get_namespaces.add_argument('--run-id', help='show test runs for the given run id', required=True) + get_namespaces.add_argument('--failed-only', action='store_true', help='show only failed test runs') get_namespaces.add_argument('--from', type=tsparser.parse, - help='Show test runs starting at the given date ' - '(MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm). Defaults to 7 days ago.', + help='show test runs starting at the given date ' + '(mm-dd-yyyy, or mm-dd-yyyy hh:mm:ss.mmmmmm). defaults to 7 days ago.', default=(datetime.today() - timedelta(days=7)).date()) def _add_describe_cli(subparsers): - describe = subparsers.add_parser('describe', help='Describe a resource') + describe = subparsers.add_parser('describe', help='describe a resource') describe.set_defaults(main=describe_object) - describe_subparsers = describe.add_subparsers(title='Resource type', dest='resource_type', required=True) - describe_runs = describe_subparsers.add_parser('runs', help='Describe a test run') - describe_runs.add_argument('test_run_id', help='Show test run details') + describe_subparsers = describe.add_subparsers(title='resource type', dest='resource_type', required=True) + describe_runs = describe_subparsers.add_parser('runs', help='describe a test run') + describe_runs.add_argument('test_run_id', help='show test run details') describe_runs.set_defaults(main=describe_object) def _add_logs_cli(subparsers): - logs = subparsers.add_parser('logs', help='Fetch pod logs') + logs = subparsers.add_parser('logs', help='fetch pod logs') logs.set_defaults(main=get_logs) - logs.add_argument('--pods', nargs='+', help='Pods to fetch logs for', required=True) - logs.add_argument('--from', dest='from_', type=tsparser.parse, - help='Show entries from date/time (MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm), ' + log_subparsers = logs.add_subparsers(title='resource type', dest='resource_type', required=True) + + logs.add_argument('--no-color', dest='no_color', action='store_true', help='disable colored output') + + pod_logs = log_subparsers.add_parser('pods', help='fetch logs for a pod') + pod_logs.add_argument('pods', nargs='+', help='pod names to fetch logs from') + pod_logs.add_argument('--from', dest='from_', type=tsparser.parse, + help='show entries from date/time (MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm), ' 'treated as UTC if no timezone given', default=None) - logs.add_argument('--to', dest='to', type=tsparser.parse, - help='Show entries until date/time (MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm), ' + pod_logs.add_argument('--to', dest='to', type=tsparser.parse, + help='show entries until date/time (MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm), ' 'treated as UTC if no timezone given', default=None) - logs.add_argument('--no-color', dest='no_color', action='store_true', help='Disable colored output') + + run_logs = log_subparsers.add_parser('runs', help='fetch logs for a test run') + run_logs.add_argument('test_run_id', help='run ID to fetch logs from') if __name__ == '__main__': diff --git a/logtools/resource/core.py b/logtools/resource/core.py index 6b3fdc1..dc58612 100644 --- a/logtools/resource/core.py +++ b/logtools/resource/core.py @@ -2,7 +2,7 @@ import abc from dataclasses import dataclass from datetime import datetime from enum import Enum -from typing import Iterator, Optional +from typing import Iterator, Optional, List @dataclass(frozen=True) @@ -26,11 +26,11 @@ class TestStatus(Enum): @dataclass(frozen=True) -class TestRun: +class SummarizedTestRun: id: str run_id: str test_name: str - pods: str + pods: List[str] start: datetime end: datetime duration: float @@ -38,8 +38,8 @@ class TestRun: @dataclass(frozen=True) -class TestRunDescription: - test_run: TestRun +class TestRun: + test_run: SummarizedTestRun error: Optional[str] stacktrace: Optional[str] @@ -54,9 +54,9 @@ class Repository(abc.ABC): ... @abc.abstractmethod - def test_runs(self, run_id: str, failed_only=False) -> Iterator[TestRun]: + def test_runs(self, run_id: str, failed_only=False) -> Iterator[SummarizedTestRun]: ... @abc.abstractmethod - def describe_test_run(self, test_run_id: str) -> TestRunDescription: + def test_run(self, test_run_id: str) -> TestRun: ... diff --git a/logtools/resource/elastic_search_log_repo.py b/logtools/resource/elastic_search_log_repo.py index 694fec4..c09453c 100644 --- a/logtools/resource/elastic_search_log_repo.py +++ b/logtools/resource/elastic_search_log_repo.py @@ -5,7 +5,7 @@ from typing import Optional, Iterator, Dict, Any from dateutil import parser from elasticsearch import Elasticsearch -from logtools.resource.core import Repository, TestRunDescription, TestStatus, TestRun, Namespace, Pod +from logtools.resource.core import Repository, TestRun, TestStatus, SummarizedTestRun, Namespace, Pod logger = logging.getLogger(__name__) @@ -92,7 +92,7 @@ class ElasticSearchLogRepo(Repository): indices=tuple(sorted(index['key'] for index in pod['indices']['buckets'])) ) - def test_runs(self, run_id: str, failed_only=False) -> Iterator[TestRun]: + def test_runs(self, run_id: str, failed_only=False) -> Iterator[SummarizedTestRun]: query = self._time_limited({ 'query': { 'bool': { @@ -109,21 +109,21 @@ class ElasticSearchLogRepo(Repository): for document in self.client.search(index=TEST_STATUS_INDEX_SET, body=query)['hits']['hits']: # type: ignore yield self._test_run_from_document(document['_index'], document['_id'], document['_source']) - def describe_test_run(self, test_run_id: str) -> TestRunDescription: + def test_run(self, test_run_id: str) -> TestRun: index, doc_id = test_run_id.split('/') document = self.client.get(index=index, id=doc_id) source = document['_source'] - return TestRunDescription( + return TestRun( test_run=self._test_run_from_document(document['_index'], document['_id'], source), error=source.get('error'), stacktrace=source.get('message'), ) @staticmethod - def _test_run_from_document(index: str, doc_id: str, source: Dict[str, str]): + def _test_run_from_document(index: str, doc_id: str, source: Dict[str, str]) -> SummarizedTestRun: start = parser.parse(source['teststart']) duration = float(source['testduration']) - return TestRun( + return SummarizedTestRun( id=f"{index}/{doc_id}", run_id=source['runid'], test_name=source['testname'], @@ -131,7 +131,7 @@ class ElasticSearchLogRepo(Repository): end=start + timedelta(seconds=duration), duration=duration, status=TestStatus(source['status'].lower()), - pods=source['involvedpods'], + pods=source['involvedpodnames'].split(','), ) def _time_limited(self, query: Dict[str, Any]) -> Dict[str, Any]: diff --git a/logtools/resource/tests/cassettes/test_elasticsearch_log_repo/test_should_retrieve_failing_test_runs_over_a_single_run_id.yaml b/logtools/resource/tests/cassettes/test_elasticsearch_log_repo/test_should_retrieve_failing_test_runs_over_a_single_run_id.yaml deleted file mode 100644 index 55808e2..0000000 --- a/logtools/resource/tests/cassettes/test_elasticsearch_log_repo/test_should_retrieve_failing_test_runs_over_a_single_run_id.yaml +++ /dev/null @@ -1,480 +0,0 @@ -interactions: -- request: - body: '{"query":{"bool":{"filter":[{"term":{"runid.keyword":"20240206-093136"}},{"term":{"status.keyword":"Failed"}}]}},"size":10000,"sort":[{"@timestamp":"desc"}]}' - headers: - accept: - - application/vnd.elasticsearch+json; compatible-with=8 - connection: - - keep-alive - content-type: - - application/vnd.elasticsearch+json; compatible-with=8 - user-agent: - - elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0) - x-elastic-client-meta: - - es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7 - method: POST - uri: http://localhost:9200/continuous-tests-status-*/_search - response: - body: - string: '{"took":5,"timed_out":false,"_shards":{"total":24,"successful":24,"skipped":0,"failed":0},"hits":{"total":{"value":1,"relation":"eq"},"max_score":null,"hits":[{"_index":"continuous-tests-status-2024.02.07","_id":"t5apg40BTe2pYqi1l9b8","_score":null,"_ignored":["message.keyword"],"_source":{"@timestamp":"2024-02-06T23:07:39.6615501Z","avgupload":"297.8KB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged - build","codexrevision":"6869475","error":" data/zDvZRwzm5UemKDPMvadCu999HrqUnCJGzvKnsF7eiy2XV3TzoW7V/network'' - timed out after 3 tries over 10 mins, 1 secs.","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":",","message":"System.Exception: - System.TimeoutException: Retry ''HTTP-GET-STREAM: data/zDvZRwzm5UemKDPMvadCu999HrqUnCJGzvKnsF7eiy2XV3TzoW7V/network'' - timed out after 3 tries over 10 mins, 1 secs.\n ---> System.AggregateException: - One or more errors occurred. (One or more errors occurred. (A task was canceled.)) - (One or more errors occurred. (A task was canceled.)) (One or more errors - occurred. (A task was canceled.)) (One or more errors occurred. (Connection - refused (bootstrap-2-2-int.two-client-test-bugfix.svc.cluster.local:30014)))\n - ---> System.AggregateException: One or more errors occurred. (A task was canceled.)\n - ---> System.Threading.Tasks.TaskCanceledException: A task was canceled.\n at - System.Threading.Tasks.Task.GetExceptions(Boolean includeTaskCanceledExceptions)\n at - System.Threading.Tasks.Task.ThrowIfExceptional(Boolean includeTaskCanceledExceptions)\n at - System.Threading.Tasks.Task.Wait(Int32 millisecondsTimeout, CancellationToken - cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at Utils.Time.Wait[T](Task`1 - task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 12\n at Core.Http.<>c__DisplayClass15_0.b__0() - in /cs-codex-dist-tests/Framework/Core/Http.cs:line 125\n at Utils.Time.Retry[T](Func`1 - action, Int32 maxRetries, TimeSpan retryTime, String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line - 144\n at Core.Http.LockRetry[T](Func`1 operation, String description) in - /cs-codex-dist-tests/Framework/Core/Http.cs:line 212\n at Core.Http.HttpGetStream(String - route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line 120\n at CodexPlugin.CodexAccess.DownloadFile(String - contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line - 71\n at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile - file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line - 200\n at CodexPlugin.CodexNode.<>c__DisplayClass35_0.b__0() - in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n at - Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug) - in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n at CodexPlugin.CodexNode.DownloadContent(ContentId - contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line - 128\n at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.b__0() - in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 43\n at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action - action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 82\n at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 43\n at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n at - System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags - invokeAttr)\n at System.Reflection.RuntimeMethodInfo.Invoke(Object obj, - BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n at - ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke) - in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n at - ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 250\n at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 150\n at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in - /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n at - ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 53\n at Utils.TaskFactory.CatchException(Action action, String name) in - /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n at Utils.TaskFactory.<>c__DisplayClass2_0.b__0() - in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread - threadPoolThread, ExecutionContext executionContext, ContextCallback callback, - Object state)\n at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task& - currentTaskSlot, Thread threadPoolThread)\n at System.Threading.ThreadPoolWorkQueue.Dispatch()\n at - System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n--- - End of stack trace from previous location ---\n\n --- End of inner exception - stack trace ---\n at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean - includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.Wait(Int32 - millisecondsTimeout, CancellationToken cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at - Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line - 12\n at Core.Http.<>c__DisplayClass15_0.b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 125\n at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime, - String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n --- - End of inner exception stack trace ---\n ---> (Inner Exception #1) System.AggregateException: - One or more errors occurred. (A task was canceled.)\n ---> System.Threading.Tasks.TaskCanceledException: - A task was canceled.\n at System.Threading.Tasks.Task.GetExceptions(Boolean - includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean - includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.Wait(Int32 - millisecondsTimeout, CancellationToken cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at - Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line - 12\n at Core.Http.<>c__DisplayClass15_0.b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 125\n at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime, - String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n at - Core.Http.LockRetry[T](Func`1 operation, String description) in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 212\n at Core.Http.HttpGetStream(String route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 120\n at CodexPlugin.CodexAccess.DownloadFile(String contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line - 71\n at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile - file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line - 200\n at CodexPlugin.CodexNode.<>c__DisplayClass35_0.b__0() - in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n at - Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug) - in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n at CodexPlugin.CodexNode.DownloadContent(ContentId - contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line - 128\n at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.b__0() - in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 43\n at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action - action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 82\n at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 43\n at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n at - System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags - invokeAttr)\n at System.Reflection.RuntimeMethodInfo.Invoke(Object obj, - BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n at - ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke) - in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n at - ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 250\n at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 150\n at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in - /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n at - ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 53\n at Utils.TaskFactory.CatchException(Action action, String name) in - /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n at Utils.TaskFactory.<>c__DisplayClass2_0.b__0() - in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread - threadPoolThread, ExecutionContext executionContext, ContextCallback callback, - Object state)\n at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task& - currentTaskSlot, Thread threadPoolThread)\n at System.Threading.ThreadPoolWorkQueue.Dispatch()\n at - System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n--- - End of stack trace from previous location ---\n\n --- End of inner exception - stack trace ---\n at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean - includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.Wait(Int32 - millisecondsTimeout, CancellationToken cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at - Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line - 12\n at Core.Http.<>c__DisplayClass15_0.b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 125\n at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime, - String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n - ---> (Inner Exception #2) System.AggregateException: One or more errors occurred. - (A task was canceled.)\n ---> System.Threading.Tasks.TaskCanceledException: - A task was canceled.\n at System.Threading.Tasks.Task.GetExceptions(Boolean - includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean - includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.Wait(Int32 - millisecondsTimeout, CancellationToken cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at - Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line - 12\n at Core.Http.<>c__DisplayClass15_0.b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 125\n at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime, - String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n at - Core.Http.LockRetry[T](Func`1 operation, String description) in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 212\n at Core.Http.HttpGetStream(String route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 120\n at CodexPlugin.CodexAccess.DownloadFile(String contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line - 71\n at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile - file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line - 200\n at CodexPlugin.CodexNode.<>c__DisplayClass35_0.b__0() - in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n at - Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug) - in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n at CodexPlugin.CodexNode.DownloadContent(ContentId - contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line - 128\n at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.b__0() - in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 43\n at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action - action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 82\n at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 43\n at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n at - System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags - invokeAttr)\n at System.Reflection.RuntimeMethodInfo.Invoke(Object obj, - BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n at - ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke) - in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n at - ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 250\n at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 150\n at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in - /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n at - ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 53\n at Utils.TaskFactory.CatchException(Action action, String name) in - /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n at Utils.TaskFactory.<>c__DisplayClass2_0.b__0() - in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread - threadPoolThread, ExecutionContext executionContext, ContextCallback callback, - Object state)\n at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task& - currentTaskSlot, Thread threadPoolThread)\n at System.Threading.ThreadPoolWorkQueue.Dispatch()\n at - System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n--- - End of stack trace from previous location ---\n\n --- End of inner exception - stack trace ---\n at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean - includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.Wait(Int32 - millisecondsTimeout, CancellationToken cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at - Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line - 12\n at Core.Http.<>c__DisplayClass15_0.b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 125\n at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime, - String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n - ---> (Inner Exception #3) System.AggregateException: One or more errors occurred. - (Connection refused (bootstrap-2-2-int.two-client-test-bugfix.svc.cluster.local:30014))\n - ---> System.Net.Http.HttpRequestException: Connection refused (bootstrap-2-2-int.two-client-test-bugfix.svc.cluster.local:30014)\n - ---> System.Net.Sockets.SocketException (111): Connection refused\n at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.ThrowException(SocketError - error, CancellationToken cancellationToken)\n at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.System.Threading.Tasks.Sources.IValueTaskSource.GetResult(Int16 - token)\n at System.Net.Sockets.Socket.g__WaitForConnectWithCancellation|281_0(AwaitableSocketAsyncEventArgs - saea, ValueTask connectTask, CancellationToken cancellationToken)\n at System.Net.Http.HttpConnectionPool.ConnectToTcpHostAsync(String - host, Int32 port, HttpRequestMessage initialRequest, Boolean async, CancellationToken - cancellationToken)\n --- End of inner exception stack trace ---\n at System.Net.Http.HttpConnectionPool.ConnectToTcpHostAsync(String - host, Int32 port, HttpRequestMessage initialRequest, Boolean async, CancellationToken - cancellationToken)\n at System.Net.Http.HttpConnectionPool.ConnectAsync(HttpRequestMessage - request, Boolean async, CancellationToken cancellationToken)\n at System.Net.Http.HttpConnectionPool.CreateHttp11ConnectionAsync(HttpRequestMessage - request, Boolean async, CancellationToken cancellationToken)\n at System.Net.Http.HttpConnectionPool.AddHttp11ConnectionAsync(QueueItem - queueItem)\n at System.Threading.Tasks.TaskCompletionSourceWithCancellation`1.WaitWithCancellationAsync(CancellationToken - cancellationToken)\n at System.Net.Http.HttpConnectionPool.HttpConnectionWaiter`1.WaitForConnectionAsync(Boolean - async, CancellationToken requestCancellationToken)\n at System.Net.Http.HttpConnectionPool.SendWithVersionDetectionAndRetryAsync(HttpRequestMessage - request, Boolean async, Boolean doRequestAuth, CancellationToken cancellationToken)\n at - System.Net.Http.RedirectHandler.SendAsync(HttpRequestMessage request, Boolean - async, CancellationToken cancellationToken)\n at System.Net.Http.HttpClient.GetStreamAsyncCore(HttpRequestMessage - request, CancellationToken cancellationToken)\n --- End of inner exception - stack trace ---\n at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean - includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.Wait(Int32 - millisecondsTimeout, CancellationToken cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at - Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line - 12\n at Core.Http.<>c__DisplayClass15_0.b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 125\n at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime, - String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n --- - End of inner exception stack trace ---\n at Utils.Time.Retry[T](Func`1 action, - Int32 maxRetries, TimeSpan retryTime, String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line - 139\n at Core.Http.LockRetry[T](Func`1 operation, String description) in - /cs-codex-dist-tests/Framework/Core/Http.cs:line 212\n at Core.Http.HttpGetStream(String - route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line 120\n at CodexPlugin.CodexAccess.DownloadFile(String - contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line - 71\n at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile - file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line - 200\n at CodexPlugin.CodexNode.<>c__DisplayClass35_0.b__0() - in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n at - Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug) - in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n at CodexPlugin.CodexNode.DownloadContent(ContentId - contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line - 128\n at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.b__0() - in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 43\n at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action - action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 82\n at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 43\n at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n at - System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags - invokeAttr)\n at ContinuousTests.SingleTestRun.ThrowFailTest() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 186\n at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 155\n at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in - /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Failed","testduration":"0","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T22:52:53.0881883Z","testtype":"continuous-tests"},"sort":[1707260859661]}]}}' - headers: - Transfer-Encoding: - - chunked - X-elastic-product: - - Elasticsearch - content-type: - - application/vnd.elasticsearch+json;compatible-with=8 - status: - code: 200 - message: OK -- request: - body: null - headers: - accept: - - application/vnd.elasticsearch+json; compatible-with=8 - connection: - - keep-alive - user-agent: - - elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0) - x-elastic-client-meta: - - es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7 - method: GET - uri: http://localhost:9200/continuous-tests-status-2024.02.07/_doc/t5apg40BTe2pYqi1l9b8 - response: - body: - string: '{"_index":"continuous-tests-status-2024.02.07","_id":"t5apg40BTe2pYqi1l9b8","_version":1,"_seq_no":211,"_primary_term":1,"found":true,"_source":{"@timestamp":"2024-02-06T23:07:39.6615501Z","avgupload":"297.8KB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged - build","codexrevision":"6869475","error":" data/zDvZRwzm5UemKDPMvadCu999HrqUnCJGzvKnsF7eiy2XV3TzoW7V/network'' - timed out after 3 tries over 10 mins, 1 secs.","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":",","message":"System.Exception: - System.TimeoutException: Retry ''HTTP-GET-STREAM: data/zDvZRwzm5UemKDPMvadCu999HrqUnCJGzvKnsF7eiy2XV3TzoW7V/network'' - timed out after 3 tries over 10 mins, 1 secs.\n ---> System.AggregateException: - One or more errors occurred. (One or more errors occurred. (A task was canceled.)) - (One or more errors occurred. (A task was canceled.)) (One or more errors - occurred. (A task was canceled.)) (One or more errors occurred. (Connection - refused (bootstrap-2-2-int.two-client-test-bugfix.svc.cluster.local:30014)))\n - ---> System.AggregateException: One or more errors occurred. (A task was canceled.)\n - ---> System.Threading.Tasks.TaskCanceledException: A task was canceled.\n at - System.Threading.Tasks.Task.GetExceptions(Boolean includeTaskCanceledExceptions)\n at - System.Threading.Tasks.Task.ThrowIfExceptional(Boolean includeTaskCanceledExceptions)\n at - System.Threading.Tasks.Task.Wait(Int32 millisecondsTimeout, CancellationToken - cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at Utils.Time.Wait[T](Task`1 - task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 12\n at Core.Http.<>c__DisplayClass15_0.b__0() - in /cs-codex-dist-tests/Framework/Core/Http.cs:line 125\n at Utils.Time.Retry[T](Func`1 - action, Int32 maxRetries, TimeSpan retryTime, String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line - 144\n at Core.Http.LockRetry[T](Func`1 operation, String description) in - /cs-codex-dist-tests/Framework/Core/Http.cs:line 212\n at Core.Http.HttpGetStream(String - route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line 120\n at CodexPlugin.CodexAccess.DownloadFile(String - contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line - 71\n at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile - file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line - 200\n at CodexPlugin.CodexNode.<>c__DisplayClass35_0.b__0() - in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n at - Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug) - in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n at CodexPlugin.CodexNode.DownloadContent(ContentId - contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line - 128\n at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.b__0() - in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 43\n at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action - action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 82\n at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 43\n at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n at - System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags - invokeAttr)\n at System.Reflection.RuntimeMethodInfo.Invoke(Object obj, - BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n at - ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke) - in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n at - ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 250\n at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 150\n at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in - /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n at - ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 53\n at Utils.TaskFactory.CatchException(Action action, String name) in - /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n at Utils.TaskFactory.<>c__DisplayClass2_0.b__0() - in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread - threadPoolThread, ExecutionContext executionContext, ContextCallback callback, - Object state)\n at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task& - currentTaskSlot, Thread threadPoolThread)\n at System.Threading.ThreadPoolWorkQueue.Dispatch()\n at - System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n--- - End of stack trace from previous location ---\n\n --- End of inner exception - stack trace ---\n at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean - includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.Wait(Int32 - millisecondsTimeout, CancellationToken cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at - Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line - 12\n at Core.Http.<>c__DisplayClass15_0.b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 125\n at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime, - String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n --- - End of inner exception stack trace ---\n ---> (Inner Exception #1) System.AggregateException: - One or more errors occurred. (A task was canceled.)\n ---> System.Threading.Tasks.TaskCanceledException: - A task was canceled.\n at System.Threading.Tasks.Task.GetExceptions(Boolean - includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean - includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.Wait(Int32 - millisecondsTimeout, CancellationToken cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at - Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line - 12\n at Core.Http.<>c__DisplayClass15_0.b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 125\n at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime, - String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n at - Core.Http.LockRetry[T](Func`1 operation, String description) in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 212\n at Core.Http.HttpGetStream(String route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 120\n at CodexPlugin.CodexAccess.DownloadFile(String contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line - 71\n at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile - file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line - 200\n at CodexPlugin.CodexNode.<>c__DisplayClass35_0.b__0() - in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n at - Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug) - in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n at CodexPlugin.CodexNode.DownloadContent(ContentId - contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line - 128\n at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.b__0() - in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 43\n at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action - action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 82\n at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 43\n at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n at - System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags - invokeAttr)\n at System.Reflection.RuntimeMethodInfo.Invoke(Object obj, - BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n at - ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke) - in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n at - ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 250\n at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 150\n at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in - /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n at - ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 53\n at Utils.TaskFactory.CatchException(Action action, String name) in - /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n at Utils.TaskFactory.<>c__DisplayClass2_0.b__0() - in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread - threadPoolThread, ExecutionContext executionContext, ContextCallback callback, - Object state)\n at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task& - currentTaskSlot, Thread threadPoolThread)\n at System.Threading.ThreadPoolWorkQueue.Dispatch()\n at - System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n--- - End of stack trace from previous location ---\n\n --- End of inner exception - stack trace ---\n at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean - includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.Wait(Int32 - millisecondsTimeout, CancellationToken cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at - Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line - 12\n at Core.Http.<>c__DisplayClass15_0.b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 125\n at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime, - String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n - ---> (Inner Exception #2) System.AggregateException: One or more errors occurred. - (A task was canceled.)\n ---> System.Threading.Tasks.TaskCanceledException: - A task was canceled.\n at System.Threading.Tasks.Task.GetExceptions(Boolean - includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean - includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.Wait(Int32 - millisecondsTimeout, CancellationToken cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at - Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line - 12\n at Core.Http.<>c__DisplayClass15_0.b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 125\n at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime, - String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n at - Core.Http.LockRetry[T](Func`1 operation, String description) in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 212\n at Core.Http.HttpGetStream(String route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 120\n at CodexPlugin.CodexAccess.DownloadFile(String contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line - 71\n at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile - file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line - 200\n at CodexPlugin.CodexNode.<>c__DisplayClass35_0.b__0() - in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n at - Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug) - in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n at CodexPlugin.CodexNode.DownloadContent(ContentId - contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line - 128\n at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.b__0() - in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 43\n at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action - action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 82\n at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 43\n at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n at - System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags - invokeAttr)\n at System.Reflection.RuntimeMethodInfo.Invoke(Object obj, - BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n at - ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke) - in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n at - ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 250\n at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 150\n at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in - /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n at - ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 53\n at Utils.TaskFactory.CatchException(Action action, String name) in - /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n at Utils.TaskFactory.<>c__DisplayClass2_0.b__0() - in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread - threadPoolThread, ExecutionContext executionContext, ContextCallback callback, - Object state)\n at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task& - currentTaskSlot, Thread threadPoolThread)\n at System.Threading.ThreadPoolWorkQueue.Dispatch()\n at - System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n--- - End of stack trace from previous location ---\n\n --- End of inner exception - stack trace ---\n at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean - includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.Wait(Int32 - millisecondsTimeout, CancellationToken cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at - Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line - 12\n at Core.Http.<>c__DisplayClass15_0.b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 125\n at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime, - String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n - ---> (Inner Exception #3) System.AggregateException: One or more errors occurred. - (Connection refused (bootstrap-2-2-int.two-client-test-bugfix.svc.cluster.local:30014))\n - ---> System.Net.Http.HttpRequestException: Connection refused (bootstrap-2-2-int.two-client-test-bugfix.svc.cluster.local:30014)\n - ---> System.Net.Sockets.SocketException (111): Connection refused\n at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.ThrowException(SocketError - error, CancellationToken cancellationToken)\n at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.System.Threading.Tasks.Sources.IValueTaskSource.GetResult(Int16 - token)\n at System.Net.Sockets.Socket.g__WaitForConnectWithCancellation|281_0(AwaitableSocketAsyncEventArgs - saea, ValueTask connectTask, CancellationToken cancellationToken)\n at System.Net.Http.HttpConnectionPool.ConnectToTcpHostAsync(String - host, Int32 port, HttpRequestMessage initialRequest, Boolean async, CancellationToken - cancellationToken)\n --- End of inner exception stack trace ---\n at System.Net.Http.HttpConnectionPool.ConnectToTcpHostAsync(String - host, Int32 port, HttpRequestMessage initialRequest, Boolean async, CancellationToken - cancellationToken)\n at System.Net.Http.HttpConnectionPool.ConnectAsync(HttpRequestMessage - request, Boolean async, CancellationToken cancellationToken)\n at System.Net.Http.HttpConnectionPool.CreateHttp11ConnectionAsync(HttpRequestMessage - request, Boolean async, CancellationToken cancellationToken)\n at System.Net.Http.HttpConnectionPool.AddHttp11ConnectionAsync(QueueItem - queueItem)\n at System.Threading.Tasks.TaskCompletionSourceWithCancellation`1.WaitWithCancellationAsync(CancellationToken - cancellationToken)\n at System.Net.Http.HttpConnectionPool.HttpConnectionWaiter`1.WaitForConnectionAsync(Boolean - async, CancellationToken requestCancellationToken)\n at System.Net.Http.HttpConnectionPool.SendWithVersionDetectionAndRetryAsync(HttpRequestMessage - request, Boolean async, Boolean doRequestAuth, CancellationToken cancellationToken)\n at - System.Net.Http.RedirectHandler.SendAsync(HttpRequestMessage request, Boolean - async, CancellationToken cancellationToken)\n at System.Net.Http.HttpClient.GetStreamAsyncCore(HttpRequestMessage - request, CancellationToken cancellationToken)\n --- End of inner exception - stack trace ---\n at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean - includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.Wait(Int32 - millisecondsTimeout, CancellationToken cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at - Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line - 12\n at Core.Http.<>c__DisplayClass15_0.b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 125\n at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime, - String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n --- - End of inner exception stack trace ---\n at Utils.Time.Retry[T](Func`1 action, - Int32 maxRetries, TimeSpan retryTime, String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line - 139\n at Core.Http.LockRetry[T](Func`1 operation, String description) in - /cs-codex-dist-tests/Framework/Core/Http.cs:line 212\n at Core.Http.HttpGetStream(String - route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line 120\n at CodexPlugin.CodexAccess.DownloadFile(String - contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line - 71\n at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile - file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line - 200\n at CodexPlugin.CodexNode.<>c__DisplayClass35_0.b__0() - in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n at - Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug) - in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n at CodexPlugin.CodexNode.DownloadContent(ContentId - contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line - 128\n at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.b__0() - in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 43\n at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action - action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 82\n at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 43\n at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n at - System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags - invokeAttr)\n at ContinuousTests.SingleTestRun.ThrowFailTest() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 186\n at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 155\n at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in - /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Failed","testduration":"0","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T22:52:53.0881883Z","testtype":"continuous-tests"}}' - headers: - X-elastic-product: - - Elasticsearch - content-length: - - '20829' - content-type: - - application/vnd.elasticsearch+json;compatible-with=8 - status: - code: 200 - message: OK -version: 1 diff --git a/logtools/resource/tests/cassettes/test_elasticsearch_log_repo/test_should_retrieve_test_runs_over_a_single_run_id.yaml b/logtools/resource/tests/cassettes/test_elasticsearch_log_repo/test_should_retrieve_test_runs_over_a_single_run_id.yaml deleted file mode 100644 index e5d9bab..0000000 --- a/logtools/resource/tests/cassettes/test_elasticsearch_log_repo/test_should_retrieve_test_runs_over_a_single_run_id.yaml +++ /dev/null @@ -1,255 +0,0 @@ -interactions: -- request: - body: '{"query":{"bool":{"filter":[{"term":{"runid.keyword":"20240206-093136"}}]}},"size":10000,"sort":[{"@timestamp":"desc"}]}' - headers: - accept: - - application/vnd.elasticsearch+json; compatible-with=8 - connection: - - keep-alive - content-type: - - application/vnd.elasticsearch+json; compatible-with=8 - user-agent: - - elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0) - x-elastic-client-meta: - - es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7 - method: POST - uri: http://localhost:9200/continuous-tests-status-*/_search - response: - body: - string: '{"took":4,"timed_out":false,"_shards":{"total":24,"successful":24,"skipped":0,"failed":0},"hits":{"total":{"value":14,"relation":"eq"},"max_score":null,"hits":[{"_index":"continuous-tests-status-2024.02.07","_id":"t5apg40BTe2pYqi1l9b8","_score":null,"_ignored":["message.keyword"],"_source":{"@timestamp":"2024-02-06T23:07:39.6615501Z","avgupload":"297.8KB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged - build","codexrevision":"6869475","error":" data/zDvZRwzm5UemKDPMvadCu999HrqUnCJGzvKnsF7eiy2XV3TzoW7V/network'' - timed out after 3 tries over 10 mins, 1 secs.","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":",","message":"System.Exception: - System.TimeoutException: Retry ''HTTP-GET-STREAM: data/zDvZRwzm5UemKDPMvadCu999HrqUnCJGzvKnsF7eiy2XV3TzoW7V/network'' - timed out after 3 tries over 10 mins, 1 secs.\n ---> System.AggregateException: - One or more errors occurred. (One or more errors occurred. (A task was canceled.)) - (One or more errors occurred. (A task was canceled.)) (One or more errors - occurred. (A task was canceled.)) (One or more errors occurred. (Connection - refused (bootstrap-2-2-int.two-client-test-bugfix.svc.cluster.local:30014)))\n - ---> System.AggregateException: One or more errors occurred. (A task was canceled.)\n - ---> System.Threading.Tasks.TaskCanceledException: A task was canceled.\n at - System.Threading.Tasks.Task.GetExceptions(Boolean includeTaskCanceledExceptions)\n at - System.Threading.Tasks.Task.ThrowIfExceptional(Boolean includeTaskCanceledExceptions)\n at - System.Threading.Tasks.Task.Wait(Int32 millisecondsTimeout, CancellationToken - cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at Utils.Time.Wait[T](Task`1 - task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 12\n at Core.Http.<>c__DisplayClass15_0.b__0() - in /cs-codex-dist-tests/Framework/Core/Http.cs:line 125\n at Utils.Time.Retry[T](Func`1 - action, Int32 maxRetries, TimeSpan retryTime, String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line - 144\n at Core.Http.LockRetry[T](Func`1 operation, String description) in - /cs-codex-dist-tests/Framework/Core/Http.cs:line 212\n at Core.Http.HttpGetStream(String - route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line 120\n at CodexPlugin.CodexAccess.DownloadFile(String - contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line - 71\n at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile - file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line - 200\n at CodexPlugin.CodexNode.<>c__DisplayClass35_0.b__0() - in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n at - Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug) - in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n at CodexPlugin.CodexNode.DownloadContent(ContentId - contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line - 128\n at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.b__0() - in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 43\n at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action - action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 82\n at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 43\n at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n at - System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags - invokeAttr)\n at System.Reflection.RuntimeMethodInfo.Invoke(Object obj, - BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n at - ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke) - in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n at - ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 250\n at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 150\n at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in - /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n at - ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 53\n at Utils.TaskFactory.CatchException(Action action, String name) in - /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n at Utils.TaskFactory.<>c__DisplayClass2_0.b__0() - in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread - threadPoolThread, ExecutionContext executionContext, ContextCallback callback, - Object state)\n at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task& - currentTaskSlot, Thread threadPoolThread)\n at System.Threading.ThreadPoolWorkQueue.Dispatch()\n at - System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n--- - End of stack trace from previous location ---\n\n --- End of inner exception - stack trace ---\n at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean - includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.Wait(Int32 - millisecondsTimeout, CancellationToken cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at - Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line - 12\n at Core.Http.<>c__DisplayClass15_0.b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 125\n at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime, - String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n --- - End of inner exception stack trace ---\n ---> (Inner Exception #1) System.AggregateException: - One or more errors occurred. (A task was canceled.)\n ---> System.Threading.Tasks.TaskCanceledException: - A task was canceled.\n at System.Threading.Tasks.Task.GetExceptions(Boolean - includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean - includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.Wait(Int32 - millisecondsTimeout, CancellationToken cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at - Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line - 12\n at Core.Http.<>c__DisplayClass15_0.b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 125\n at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime, - String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n at - Core.Http.LockRetry[T](Func`1 operation, String description) in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 212\n at Core.Http.HttpGetStream(String route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 120\n at CodexPlugin.CodexAccess.DownloadFile(String contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line - 71\n at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile - file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line - 200\n at CodexPlugin.CodexNode.<>c__DisplayClass35_0.b__0() - in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n at - Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug) - in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n at CodexPlugin.CodexNode.DownloadContent(ContentId - contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line - 128\n at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.b__0() - in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 43\n at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action - action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 82\n at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 43\n at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n at - System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags - invokeAttr)\n at System.Reflection.RuntimeMethodInfo.Invoke(Object obj, - BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n at - ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke) - in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n at - ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 250\n at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 150\n at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in - /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n at - ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 53\n at Utils.TaskFactory.CatchException(Action action, String name) in - /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n at Utils.TaskFactory.<>c__DisplayClass2_0.b__0() - in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread - threadPoolThread, ExecutionContext executionContext, ContextCallback callback, - Object state)\n at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task& - currentTaskSlot, Thread threadPoolThread)\n at System.Threading.ThreadPoolWorkQueue.Dispatch()\n at - System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n--- - End of stack trace from previous location ---\n\n --- End of inner exception - stack trace ---\n at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean - includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.Wait(Int32 - millisecondsTimeout, CancellationToken cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at - Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line - 12\n at Core.Http.<>c__DisplayClass15_0.b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 125\n at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime, - String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n - ---> (Inner Exception #2) System.AggregateException: One or more errors occurred. - (A task was canceled.)\n ---> System.Threading.Tasks.TaskCanceledException: - A task was canceled.\n at System.Threading.Tasks.Task.GetExceptions(Boolean - includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean - includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.Wait(Int32 - millisecondsTimeout, CancellationToken cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at - Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line - 12\n at Core.Http.<>c__DisplayClass15_0.b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 125\n at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime, - String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n at - Core.Http.LockRetry[T](Func`1 operation, String description) in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 212\n at Core.Http.HttpGetStream(String route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 120\n at CodexPlugin.CodexAccess.DownloadFile(String contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line - 71\n at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile - file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line - 200\n at CodexPlugin.CodexNode.<>c__DisplayClass35_0.b__0() - in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n at - Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug) - in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n at CodexPlugin.CodexNode.DownloadContent(ContentId - contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line - 128\n at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.b__0() - in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 43\n at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action - action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 82\n at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 43\n at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n at - System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags - invokeAttr)\n at System.Reflection.RuntimeMethodInfo.Invoke(Object obj, - BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n at - ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke) - in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n at - ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 250\n at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 150\n at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in - /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n at - ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 53\n at Utils.TaskFactory.CatchException(Action action, String name) in - /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n at Utils.TaskFactory.<>c__DisplayClass2_0.b__0() - in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread - threadPoolThread, ExecutionContext executionContext, ContextCallback callback, - Object state)\n at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task& - currentTaskSlot, Thread threadPoolThread)\n at System.Threading.ThreadPoolWorkQueue.Dispatch()\n at - System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n--- - End of stack trace from previous location ---\n\n --- End of inner exception - stack trace ---\n at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean - includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.Wait(Int32 - millisecondsTimeout, CancellationToken cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at - Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line - 12\n at Core.Http.<>c__DisplayClass15_0.b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 125\n at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime, - String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n - ---> (Inner Exception #3) System.AggregateException: One or more errors occurred. - (Connection refused (bootstrap-2-2-int.two-client-test-bugfix.svc.cluster.local:30014))\n - ---> System.Net.Http.HttpRequestException: Connection refused (bootstrap-2-2-int.two-client-test-bugfix.svc.cluster.local:30014)\n - ---> System.Net.Sockets.SocketException (111): Connection refused\n at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.ThrowException(SocketError - error, CancellationToken cancellationToken)\n at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.System.Threading.Tasks.Sources.IValueTaskSource.GetResult(Int16 - token)\n at System.Net.Sockets.Socket.g__WaitForConnectWithCancellation|281_0(AwaitableSocketAsyncEventArgs - saea, ValueTask connectTask, CancellationToken cancellationToken)\n at System.Net.Http.HttpConnectionPool.ConnectToTcpHostAsync(String - host, Int32 port, HttpRequestMessage initialRequest, Boolean async, CancellationToken - cancellationToken)\n --- End of inner exception stack trace ---\n at System.Net.Http.HttpConnectionPool.ConnectToTcpHostAsync(String - host, Int32 port, HttpRequestMessage initialRequest, Boolean async, CancellationToken - cancellationToken)\n at System.Net.Http.HttpConnectionPool.ConnectAsync(HttpRequestMessage - request, Boolean async, CancellationToken cancellationToken)\n at System.Net.Http.HttpConnectionPool.CreateHttp11ConnectionAsync(HttpRequestMessage - request, Boolean async, CancellationToken cancellationToken)\n at System.Net.Http.HttpConnectionPool.AddHttp11ConnectionAsync(QueueItem - queueItem)\n at System.Threading.Tasks.TaskCompletionSourceWithCancellation`1.WaitWithCancellationAsync(CancellationToken - cancellationToken)\n at System.Net.Http.HttpConnectionPool.HttpConnectionWaiter`1.WaitForConnectionAsync(Boolean - async, CancellationToken requestCancellationToken)\n at System.Net.Http.HttpConnectionPool.SendWithVersionDetectionAndRetryAsync(HttpRequestMessage - request, Boolean async, Boolean doRequestAuth, CancellationToken cancellationToken)\n at - System.Net.Http.RedirectHandler.SendAsync(HttpRequestMessage request, Boolean - async, CancellationToken cancellationToken)\n at System.Net.Http.HttpClient.GetStreamAsyncCore(HttpRequestMessage - request, CancellationToken cancellationToken)\n --- End of inner exception - stack trace ---\n at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean - includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.Wait(Int32 - millisecondsTimeout, CancellationToken cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at - Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line - 12\n at Core.Http.<>c__DisplayClass15_0.b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line - 125\n at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime, - String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n --- - End of inner exception stack trace ---\n at Utils.Time.Retry[T](Func`1 action, - Int32 maxRetries, TimeSpan retryTime, String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line - 139\n at Core.Http.LockRetry[T](Func`1 operation, String description) in - /cs-codex-dist-tests/Framework/Core/Http.cs:line 212\n at Core.Http.HttpGetStream(String - route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line 120\n at CodexPlugin.CodexAccess.DownloadFile(String - contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line - 71\n at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile - file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line - 200\n at CodexPlugin.CodexNode.<>c__DisplayClass35_0.b__0() - in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n at - Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug) - in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n at CodexPlugin.CodexNode.DownloadContent(ContentId - contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line - 128\n at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.b__0() - in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 43\n at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action - action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 82\n at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line - 43\n at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n at - System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags - invokeAttr)\n at ContinuousTests.SingleTestRun.ThrowFailTest() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 186\n at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line - 155\n at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in - /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Failed","testduration":"0","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T22:52:53.0881883Z","testtype":"continuous-tests"},"sort":[1707260859661]},{"_index":"continuous-tests-status-2024.02.07","_id":"tpapg40BTe2pYqi1l9b8","_score":null,"_source":{"@timestamp":"2024-02-06T22:52:52.2996375Z","avgdownload":"780.8KB/s","avgupload":"3MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged - build","codexrevision":"6869475","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":",","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Passed","testduration":"141.9114191","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T22:50:30.3878431Z","testtype":"continuous-tests"},"sort":[1707259972299]},{"_index":"continuous-tests-status-2024.02.07","_id":"sZapg40BTe2pYqi1l9b8","_score":null,"_source":{"@timestamp":"2024-02-06T22:50:30.0675681Z","avgdownload":"152.9KB/s","avgupload":"3.1MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged - build","codexrevision":"6869475","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":",","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Passed","testduration":"572.9451525","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T22:40:57.1222113Z","testtype":"continuous-tests"},"sort":[1707259830067]},{"_index":"continuous-tests-status-2024.02.07","_id":"sJapg40BTe2pYqi1l9b8","_score":null,"_source":{"@timestamp":"2024-02-06T22:40:56.1120614Z","avgdownload":"70KB/s","avgupload":"3.1MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged - build","codexrevision":"6869475","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":",","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Passed","testduration":"1207.5289232","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T22:20:48.5829991Z","testtype":"continuous-tests"},"sort":[1707259256112]},{"_index":"continuous-tests-status-2024.02.07","_id":"r5apg40BTe2pYqi1l9b8","_score":null,"_source":{"@timestamp":"2024-02-06T22:20:47.5015704Z","avgdownload":"107KB/s","avgupload":"2.9MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged - build","codexrevision":"6869475","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":",","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Passed","testduration":"804.6780181","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T22:07:22.8233627Z","testtype":"continuous-tests"},"sort":[1707258047501]},{"_index":"continuous-tests-status-2024.02.07","_id":"rpapg40BTe2pYqi1l9b8","_score":null,"_source":{"@timestamp":"2024-02-06T22:07:13.1890259Z","avgdownload":"1.2MB/s","avgupload":"2.5MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged - build","codexrevision":"6869475","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":",","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Passed","testduration":"113.1299709","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T22:05:20.0552568Z","testtype":"continuous-tests"},"sort":[1707257233189]},{"_index":"continuous-tests-status-2024.02.07","_id":"qJapg40BTe2pYqi1l9b8","_score":null,"_source":{"@timestamp":"2024-02-06T22:05:19.1233015Z","avgdownload":"128.4KB/s","avgupload":"2.3MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged - build","codexrevision":"6869475","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":",","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Passed","testduration":"683.6157221","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T21:53:55.5063178Z","testtype":"continuous-tests"},"sort":[1707257119123]},{"_index":"continuous-tests-status-2024.02.07","_id":"p5apg40BTe2pYqi1l9b8","_score":null,"_source":{"@timestamp":"2024-02-06T21:53:47.1782269Z","avgdownload":"1MB/s","avgupload":"3.4MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged - build","codexrevision":"6869475","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":",","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Passed","testduration":"113.2005889","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T21:51:53.9774688Z","testtype":"continuous-tests"},"sort":[1707256427178]},{"_index":"continuous-tests-status-2024.02.07","_id":"ppapg40BTe2pYqi1l9b8","_score":null,"_source":{"@timestamp":"2024-02-06T21:51:45.5038683Z","avgdownload":"988.9KB/s","avgupload":"4.2MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged - build","codexrevision":"6869475","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":",","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Passed","testduration":"112.4111551","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T21:49:53.0925544Z","testtype":"continuous-tests"},"sort":[1707256305503]},{"_index":"continuous-tests-status-2024.02.07","_id":"pZapg40BTe2pYqi1l9b8","_score":null,"_source":{"@timestamp":"2024-02-06T21:49:11.8557349Z","avgdownload":"1.8MB/s","avgupload":"3.3MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged - build","codexrevision":"6869475","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":",","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Passed","testduration":"78.958915","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T21:47:52.8966341Z","testtype":"continuous-tests"},"sort":[1707256151855]},{"_index":"continuous-tests-status-2024.02.07","_id":"mpapg40BTe2pYqi1l9b8","_score":null,"_source":{"@timestamp":"2024-02-06T21:47:33.8366307Z","avgdownload":"1.1MB/s","avgupload":"4.3MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged - build","codexrevision":"6869475","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":",","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Passed","testduration":"101.1595982","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T21:45:52.6769016Z","testtype":"continuous-tests"},"sort":[1707256053836]},{"_index":"continuous-tests-status-2024.02.07","_id":"mZapg40BTe2pYqi1l9b8","_score":null,"_source":{"@timestamp":"2024-02-06T21:45:08.0812184Z","avgdownload":"1.7MB/s","avgupload":"4.1MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged - build","codexrevision":"6869475","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":",","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Passed","testduration":"75.6624344","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T21:43:52.4185559Z","testtype":"continuous-tests"},"sort":[1707255908081]},{"_index":"continuous-tests-status-2024.02.07","_id":"mJapg40BTe2pYqi1l9b8","_score":null,"_source":{"@timestamp":"2024-02-06T21:43:14.2974430Z","avgdownload":"1.8MB/s","avgupload":"3.1MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged - build","codexrevision":"6869475","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":",","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Passed","testduration":"82.0906635","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T21:41:52.2061498Z","testtype":"continuous-tests"},"sort":[1707255794297]},{"_index":"continuous-tests-status-2024.02.07","_id":"l5apg40BTe2pYqi1l9b8","_score":null,"_source":{"@timestamp":"2024-02-06T21:41:05.3802809Z","avgdownload":"1.8MB/s","avgupload":"4.4MB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged - build","codexrevision":"6869475","error":"OK","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":",","message":"OK","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Passed","testduration":"73.3825081","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T21:39:51.9897720Z","testtype":"continuous-tests"},"sort":[1707255665380]}]}}' - headers: - Transfer-Encoding: - - chunked - X-elastic-product: - - Elasticsearch - content-type: - - application/vnd.elasticsearch+json;compatible-with=8 - status: - code: 200 - message: OK -version: 1 diff --git a/logtools/resource/tests/test_elasticsearch_log_repo.py b/logtools/resource/tests/test_elasticsearch_log_repo.py index 983a587..63ec310 100644 --- a/logtools/resource/tests/test_elasticsearch_log_repo.py +++ b/logtools/resource/tests/test_elasticsearch_log_repo.py @@ -85,7 +85,7 @@ def test_should_respect_time_horizon_for_retrieving_resources(): @pytest.mark.vcr def test_should_retrieve_test_runs_over_a_single_run_id(): repo = ElasticSearchLogRepo() - runs = list(repo.test_runs('20240206-093136')) + runs = list(repo.test_runs('20240208-044040')) assert len(runs) == 14 @@ -93,10 +93,10 @@ def test_should_retrieve_test_runs_over_a_single_run_id(): @pytest.mark.vcr def test_should_retrieve_failing_test_runs_over_a_single_run_id(): repo = ElasticSearchLogRepo() - runs = list(repo.test_runs('20240206-093136', failed_only=True)) + runs = list(repo.test_runs('20240208-044040', failed_only=True)) assert len(runs) == 1 - assert repo.describe_test_run(runs[0].id).error.strip() == ( + assert repo.test_run(runs[0].id).error.strip() == ( "data/zDvZRwzm5UemKDPMvadCu999HrqUnCJGzvKnsF7eiy2XV3TzoW7V/network' timed out after " "3 tries over 10 mins, 1 secs." ) diff --git a/pyproject.toml b/pyproject.toml index e0189e1..0e914ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "logtools" -version = "1.1.4" +version = "1.2.0" description = "" authors = ["gmega "] readme = "README.md"