add base integration for test runs

This commit is contained in:
gmega 2024-02-08 11:23:56 -03:00
parent 6f66a58ede
commit 36a4992b57
No known key found for this signature in database
GPG Key ID: FFD8DAF00660270F
17 changed files with 476 additions and 108 deletions

View File

@ -1,58 +1,88 @@
import datetime
import math
import dataclasses
import os
import textwrap
from argparse import ArgumentParser
from datetime import timedelta
from datetime import timedelta, datetime
from enum import Enum
from typing import List, Iterable, cast
from json import JSONEncoder
from typing import List, Iterable, Any
import rich
from colored import Style
from dateutil import parser as tsparser
from elasticsearch import Elasticsearch
from prettytable import PrettyTable
from rich import json
from rich.console import Console
from rich.json import JSON
from rich.table import Table
from logtools.cli.palettes import ColorMap
from logtools.log.sources.input.elastic_search.elastic_search_log_repo import ElasticSearchLogRepo
from logtools.log.sources.input.elastic_search.elastic_search_source import ElasticSearchSource
from logtools.log.sources.input.elastic_search_source import ElasticSearchSource
from logtools.resource.elastic_search_log_repo import ElasticSearchLogRepo
class ResourceType(Enum):
pods = 'pods'
namespaces = 'namespaces'
runs = 'runs'
GETTERS = {
ResourceType.pods: lambda repo, args: repo.pods(prefix=args.prefix, run_id=args.run_id),
ResourceType.namespaces: lambda repo, args: repo.namespaces(prefix=args.prefix)
ResourceType.namespaces: lambda repo, args: repo.namespaces(prefix=args.prefix),
ResourceType.runs: lambda repo, args: repo.test_runs(run_id=args.run_id, failed_only=args.failed_only),
}
DESCRIBERS = {
ResourceType.runs: lambda repo, args: repo.describe_test_run(test_run_id=args.test_run_id),
}
def format_table(objects: List) -> str:
tbl = PrettyTable()
wrapper = None
def format_table(objects: List, title: str = 'Results') -> Table:
tbl = Table(title=title)
field_names = None
for obj in objects:
if not tbl.field_names:
tbl.field_names = obj.__annotations__.keys()
wrapper = textwrap.TextWrapper(width=math.ceil(os.get_terminal_size().columns / len(tbl.field_names)),
break_long_words=False)
if field_names is None:
field_names = obj.__annotations__.keys()
for field_name in field_names:
tbl.add_column(field_name, justify='left')
tbl.add_row([cast(textwrap.TextWrapper, wrapper).fill(
_format_field(getattr(obj, field))) for field in tbl.field_names])
tbl.add_row(*[_format_field(getattr(obj, field)) for field in field_names])
return tbl.get_string()
return tbl
def _format_field(field: str | Iterable[object]):
def format_json(obj: Any) -> JSON:
# For now, this is rather rudimentary.
class DataclassEncoder(JSONEncoder):
def default(self, o):
if dataclasses.is_dataclass(o):
return dataclasses.asdict(o)
elif isinstance(o, datetime):
return o.isoformat()
elif isinstance(o, Enum):
return o.value
return super().default(o)
return JSON(json.dumps(obj, cls=DataclassEncoder))
def _format_field(field: Any):
if isinstance(field, str):
return field
return ', '.join([str(item) for item in field])
elif isinstance(field, Iterable):
return ', '.join([_format_field(item) for item in field])
return str(field)
def get_object(args, client: Elasticsearch):
repo = ElasticSearchLogRepo(client=client)
print(format_table(GETTERS[ResourceType[args.resource_type]](repo, args)))
Console().print(format_table(GETTERS[ResourceType[args.resource_type]](repo, args)))
def describe_object(args, client: Elasticsearch):
repo = ElasticSearchLogRepo(client=client)
Console().print(format_json([DESCRIBERS[ResourceType[args.resource_type]](repo, args)]))
def get_logs(args, client: Elasticsearch):
@ -78,12 +108,21 @@ def main():
)
subparsers = parser.add_subparsers(title='Command', required=True)
_add_get_cli(subparsers)
_add_describe_cli(subparsers)
_add_logs_cli(subparsers)
args = parser.parse_args()
client = Elasticsearch(args.es_host, request_timeout=60)
args.main(args, client)
def _add_get_cli(subparsers):
get = subparsers.add_parser('get', help='Display existing resources')
get.add_argument('--from', type=tsparser.parse,
help='Show resources present in log messages starting at the given date '
'(MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm). Defaults to 7 days ago.',
default=(datetime.datetime.today() - timedelta(days=7)).date())
default=(datetime.today() - timedelta(days=7)).date())
get.set_defaults(main=get_object)
get_subparsers = get.add_subparsers(title='Resource type', dest='resource_type', required=True)
@ -94,6 +133,26 @@ def main():
get_namespaces = get_subparsers.add_parser('namespaces', help='Display existing namespaces')
get_namespaces.add_argument('--prefix', help='Filter namespaces by prefix')
get_namespaces = get_subparsers.add_parser('runs', help='Display current test runs')
get_namespaces.add_argument('--run-id', help='Show test runs for the given run id', required=True)
get_namespaces.add_argument('--failed-only', action='store_true', help='Show only failed test runs')
get_namespaces.add_argument('--from', type=tsparser.parse,
help='Show test runs starting at the given date '
'(MM-DD-YYYY, or MM-DD-YYYY HH:MM:SS.mmmmmm). Defaults to 7 days ago.',
default=(datetime.today() - timedelta(days=7)).date())
def _add_describe_cli(subparsers):
describe = subparsers.add_parser('describe', help='Describe a resource')
describe.set_defaults(main=describe_object)
describe_subparsers = describe.add_subparsers(title='Resource type', dest='resource_type', required=True)
describe_runs = describe_subparsers.add_parser('runs', help='Describe a test run')
describe_runs.add_argument('test_run_id', help='Show test run details')
describe_runs.set_defaults(main=describe_object)
def _add_logs_cli(subparsers):
logs = subparsers.add_parser('logs', help='Fetch pod logs')
logs.set_defaults(main=get_logs)
@ -106,12 +165,6 @@ def main():
'treated as UTC if no timezone given', default=None)
logs.add_argument('--no-color', dest='no_color', action='store_true', help='Disable colored output')
args = parser.parse_args()
client = Elasticsearch(args.es_host, request_timeout=60)
args.main(args, client)
if __name__ == '__main__':
main()

View File

@ -3,7 +3,7 @@ from datetime import timedelta
import pytest
from dateutil import parser
from logtools.log.sources.input.elastic_search.elastic_search_source import ElasticSearchSource
from logtools.log.sources.input.elastic_search_source import ElasticSearchSource
@pytest.mark.vcr

62
logtools/resource/core.py Normal file
View File

@ -0,0 +1,62 @@
import abc
from dataclasses import dataclass
from datetime import datetime
from enum import Enum
from typing import Iterator, Optional
@dataclass(frozen=True)
class Namespace:
name: str
run_id: tuple[str, ...]
indices: tuple[str, ...]
@dataclass(frozen=True)
class Pod:
name: str
namespace: str
run_id: str
indices: tuple[str, ...]
class TestStatus(Enum):
passed = 'passed'
failed = 'failed'
@dataclass(frozen=True)
class TestRun:
id: str
run_id: str
test_name: str
pods: str
start: datetime
end: datetime
duration: float
status: TestStatus
@dataclass(frozen=True)
class TestRunDescription:
test_run: TestRun
error: Optional[str]
stacktrace: Optional[str]
class Repository(abc.ABC):
@abc.abstractmethod
def namespaces(self, prefix: Optional[str] = None) -> Iterator[Namespace]:
...
@abc.abstractmethod
def pods(self, prefix: Optional[str] = None, run_id: Optional[str] = None):
...
@abc.abstractmethod
def test_runs(self, run_id: str, failed_only=False) -> Iterator[TestRun]:
...
@abc.abstractmethod
def describe_test_run(self, test_run_id: str) -> TestRunDescription:
...

View File

@ -1,56 +1,22 @@
import logging
from dataclasses import dataclass
from datetime import datetime, timedelta
from enum import Enum
from typing import Optional, Iterator, Dict, Any
from dateutil import parser
from elasticsearch import Elasticsearch
from logtools.resource.core import Repository, TestRunDescription, TestStatus, TestRun, Namespace, Pod
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class Namespace:
name: str
run_id: tuple[str, ...]
indices: tuple[str, ...]
@dataclass(frozen=True)
class Pod:
name: str
namespace: str
run_id: str
indices: tuple[str, ...]
class TestStatus(Enum):
passed = 'passed'
failed = 'failed'
@dataclass(frozen=True)
class TestRun:
id: str
run_id: str
test_name: str
pods: str
start: datetime
end: datetime
duration: float
status: TestStatus
error: Optional[str]
stacktrace: Optional[str]
MAX_AGGREGATION_BUCKETS = 1000
POD_LOGS_INDEX_SET = 'continuous-tests-pods-*'
TEST_STATUS_INDEX_SET = 'continuous-tests-status-*'
class ElasticSearchLogRepo:
class ElasticSearchLogRepo(Repository):
def __init__(
self,
client: Optional[Elasticsearch] = None,
@ -141,21 +107,32 @@ class ElasticSearchLogRepo:
query['query']['bool']['filter'].append({'term': {'status.keyword': 'Failed'}})
for document in self.client.search(index=TEST_STATUS_INDEX_SET, body=query)['hits']['hits']: # type: ignore
content = document['_source']
start = parser.parse(content['teststart'])
duration = float(content['testduration'])
yield TestRun(
id=document['_id'],
run_id=content['runid'],
test_name=content['testname'],
start=start,
end=start + timedelta(seconds=duration),
duration=duration,
status=TestStatus(content['status'].lower()),
pods=content['involvedpods'],
error=content.get('error'),
stacktrace=content.get('message')
)
yield self._test_run_from_document(document['_index'], document['_id'], document['_source'])
def describe_test_run(self, test_run_id: str) -> TestRunDescription:
index, doc_id = test_run_id.split('/')
document = self.client.get(index=index, id=doc_id)
source = document['_source']
return TestRunDescription(
test_run=self._test_run_from_document(document['_index'], document['_id'], source),
error=source.get('error'),
stacktrace=source.get('message'),
)
@staticmethod
def _test_run_from_document(index: str, doc_id: str, source: Dict[str, str]):
start = parser.parse(source['teststart'])
duration = float(source['testduration'])
return TestRun(
id=f"{index}/{doc_id}",
run_id=source['runid'],
test_name=source['testname'],
start=start,
end=start + timedelta(seconds=duration),
duration=duration,
status=TestStatus(source['status'].lower()),
pods=source['involvedpods'],
)
def _time_limited(self, query: Dict[str, Any]) -> Dict[str, Any]:
if self.since is not None:

View File

@ -16,7 +16,7 @@ interactions:
uri: http://localhost:9200/continuous-tests-status-*/_search
response:
body:
string: '{"took":4,"timed_out":false,"_shards":{"total":23,"successful":23,"skipped":0,"failed":0},"hits":{"total":{"value":1,"relation":"eq"},"max_score":null,"hits":[{"_index":"continuous-tests-status-2024.02.07","_id":"t5apg40BTe2pYqi1l9b8","_score":null,"_ignored":["message.keyword"],"_source":{"@timestamp":"2024-02-06T23:07:39.6615501Z","avgupload":"297.8KB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
string: '{"took":5,"timed_out":false,"_shards":{"total":24,"successful":24,"skipped":0,"failed":0},"hits":{"total":{"value":1,"relation":"eq"},"max_score":null,"hits":[{"_index":"continuous-tests-status-2024.02.07","_id":"t5apg40BTe2pYqi1l9b8","_score":null,"_ignored":["message.keyword"],"_source":{"@timestamp":"2024-02-06T23:07:39.6615501Z","avgupload":"297.8KB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
build","codexrevision":"6869475","error":" data/zDvZRwzm5UemKDPMvadCu999HrqUnCJGzvKnsF7eiy2XV3TzoW7V/network''
timed out after 3 tries over 10 mins, 1 secs.","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":"<CODEX24>,<BOOTSTRAP2>","message":"System.Exception:
System.TimeoutException: Retry ''HTTP-GET-STREAM: data/zDvZRwzm5UemKDPMvadCu999HrqUnCJGzvKnsF7eiy2XV3TzoW7V/network''
@ -239,4 +239,242 @@ interactions:
status:
code: 200
message: OK
- request:
body: null
headers:
accept:
- application/vnd.elasticsearch+json; compatible-with=8
connection:
- keep-alive
user-agent:
- elasticsearch-py/8.10.1 (Python/3.11.5; elastic-transport/8.10.0)
x-elastic-client-meta:
- es=8.10.1,py=3.11.5,t=8.10.0,ur=2.0.7
method: GET
uri: http://localhost:9200/continuous-tests-status-2024.02.07/_doc/t5apg40BTe2pYqi1l9b8
response:
body:
string: '{"_index":"continuous-tests-status-2024.02.07","_id":"t5apg40BTe2pYqi1l9b8","_version":1,"_seq_no":211,"_primary_term":1,"found":true,"_source":{"@timestamp":"2024-02-06T23:07:39.6615501Z","avgupload":"297.8KB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
build","codexrevision":"6869475","error":" data/zDvZRwzm5UemKDPMvadCu999HrqUnCJGzvKnsF7eiy2XV3TzoW7V/network''
timed out after 3 tries over 10 mins, 1 secs.","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":"<CODEX24>,<BOOTSTRAP2>","message":"System.Exception:
System.TimeoutException: Retry ''HTTP-GET-STREAM: data/zDvZRwzm5UemKDPMvadCu999HrqUnCJGzvKnsF7eiy2XV3TzoW7V/network''
timed out after 3 tries over 10 mins, 1 secs.\n ---> System.AggregateException:
One or more errors occurred. (One or more errors occurred. (A task was canceled.))
(One or more errors occurred. (A task was canceled.)) (One or more errors
occurred. (A task was canceled.)) (One or more errors occurred. (Connection
refused (bootstrap-2-2-int.two-client-test-bugfix.svc.cluster.local:30014)))\n
---> System.AggregateException: One or more errors occurred. (A task was canceled.)\n
---> System.Threading.Tasks.TaskCanceledException: A task was canceled.\n at
System.Threading.Tasks.Task.GetExceptions(Boolean includeTaskCanceledExceptions)\n at
System.Threading.Tasks.Task.ThrowIfExceptional(Boolean includeTaskCanceledExceptions)\n at
System.Threading.Tasks.Task.Wait(Int32 millisecondsTimeout, CancellationToken
cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at Utils.Time.Wait[T](Task`1
task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 12\n at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0()
in /cs-codex-dist-tests/Framework/Core/Http.cs:line 125\n at Utils.Time.Retry[T](Func`1
action, Int32 maxRetries, TimeSpan retryTime, String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
144\n at Core.Http.LockRetry[T](Func`1 operation, String description) in
/cs-codex-dist-tests/Framework/Core/Http.cs:line 212\n at Core.Http.HttpGetStream(String
route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line 120\n at CodexPlugin.CodexAccess.DownloadFile(String
contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line
71\n at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile
file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
200\n at CodexPlugin.CodexNode.<>c__DisplayClass35_0.<DownloadContent>b__0()
in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n at
Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug)
in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n at CodexPlugin.CodexNode.DownloadContent(ContentId
contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
128\n at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.<DownloadTestFile>b__0()
in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
43\n at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action
action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
82\n at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
43\n at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n at
System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
invokeAttr)\n at System.Reflection.RuntimeMethodInfo.Invoke(Object obj,
BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n at
ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke)
in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n at
ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
250\n at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
150\n at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
/cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n at
ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.<Run>b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
53\n at Utils.TaskFactory.CatchException(Action action, String name) in
/cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n at Utils.TaskFactory.<>c__DisplayClass2_0.<Run>b__0()
in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread
threadPoolThread, ExecutionContext executionContext, ContextCallback callback,
Object state)\n at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task&
currentTaskSlot, Thread threadPoolThread)\n at System.Threading.ThreadPoolWorkQueue.Dispatch()\n at
System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n---
End of stack trace from previous location ---\n\n --- End of inner exception
stack trace ---\n at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.Wait(Int32
millisecondsTimeout, CancellationToken cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at
Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
12\n at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
125\n at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n ---
End of inner exception stack trace ---\n ---> (Inner Exception #1) System.AggregateException:
One or more errors occurred. (A task was canceled.)\n ---> System.Threading.Tasks.TaskCanceledException:
A task was canceled.\n at System.Threading.Tasks.Task.GetExceptions(Boolean
includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.Wait(Int32
millisecondsTimeout, CancellationToken cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at
Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
12\n at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
125\n at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n at
Core.Http.LockRetry[T](Func`1 operation, String description) in /cs-codex-dist-tests/Framework/Core/Http.cs:line
212\n at Core.Http.HttpGetStream(String route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line
120\n at CodexPlugin.CodexAccess.DownloadFile(String contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line
71\n at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile
file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
200\n at CodexPlugin.CodexNode.<>c__DisplayClass35_0.<DownloadContent>b__0()
in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n at
Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug)
in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n at CodexPlugin.CodexNode.DownloadContent(ContentId
contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
128\n at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.<DownloadTestFile>b__0()
in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
43\n at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action
action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
82\n at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
43\n at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n at
System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
invokeAttr)\n at System.Reflection.RuntimeMethodInfo.Invoke(Object obj,
BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n at
ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke)
in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n at
ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
250\n at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
150\n at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
/cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n at
ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.<Run>b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
53\n at Utils.TaskFactory.CatchException(Action action, String name) in
/cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n at Utils.TaskFactory.<>c__DisplayClass2_0.<Run>b__0()
in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread
threadPoolThread, ExecutionContext executionContext, ContextCallback callback,
Object state)\n at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task&
currentTaskSlot, Thread threadPoolThread)\n at System.Threading.ThreadPoolWorkQueue.Dispatch()\n at
System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n---
End of stack trace from previous location ---\n\n --- End of inner exception
stack trace ---\n at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.Wait(Int32
millisecondsTimeout, CancellationToken cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at
Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
12\n at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
125\n at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n
---> (Inner Exception #2) System.AggregateException: One or more errors occurred.
(A task was canceled.)\n ---> System.Threading.Tasks.TaskCanceledException:
A task was canceled.\n at System.Threading.Tasks.Task.GetExceptions(Boolean
includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.Wait(Int32
millisecondsTimeout, CancellationToken cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at
Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
12\n at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
125\n at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144\n at
Core.Http.LockRetry[T](Func`1 operation, String description) in /cs-codex-dist-tests/Framework/Core/Http.cs:line
212\n at Core.Http.HttpGetStream(String route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line
120\n at CodexPlugin.CodexAccess.DownloadFile(String contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line
71\n at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile
file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
200\n at CodexPlugin.CodexNode.<>c__DisplayClass35_0.<DownloadContent>b__0()
in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n at
Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug)
in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n at CodexPlugin.CodexNode.DownloadContent(ContentId
contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
128\n at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.<DownloadTestFile>b__0()
in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
43\n at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action
action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
82\n at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
43\n at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n at
System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
invokeAttr)\n at System.Reflection.RuntimeMethodInfo.Invoke(Object obj,
BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture)\n at
ContinuousTests.TestHandle.InvokeMoment(Int32 currentMoment, Action`1 beforeInvoke)
in /cs-codex-dist-tests/Tests/CodexContinuousTests/TestHandle.cs:line 47\n at
ContinuousTests.SingleTestRun.RunMoment(Int32 t) in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
250\n at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
150\n at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
/cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76\n at
ContinuousTests.SingleTestRun.<>c__DisplayClass13_0.<Run>b__0() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
53\n at Utils.TaskFactory.CatchException(Action action, String name) in
/cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 20\n at Utils.TaskFactory.<>c__DisplayClass2_0.<Run>b__0()
in /cs-codex-dist-tests/Framework/Utils/TaskFactory.cs:line 12\n at System.Threading.ExecutionContext.RunFromThreadPoolDispatchLoop(Thread
threadPoolThread, ExecutionContext executionContext, ContextCallback callback,
Object state)\n at System.Threading.Tasks.Task.ExecuteWithThreadLocal(Task&
currentTaskSlot, Thread threadPoolThread)\n at System.Threading.ThreadPoolWorkQueue.Dispatch()\n at
System.Threading.PortableThreadPool.WorkerThread.WorkerThreadStart()\n---
End of stack trace from previous location ---\n\n --- End of inner exception
stack trace ---\n at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.Wait(Int32
millisecondsTimeout, CancellationToken cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at
Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
12\n at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
125\n at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n
---> (Inner Exception #3) System.AggregateException: One or more errors occurred.
(Connection refused (bootstrap-2-2-int.two-client-test-bugfix.svc.cluster.local:30014))\n
---> System.Net.Http.HttpRequestException: Connection refused (bootstrap-2-2-int.two-client-test-bugfix.svc.cluster.local:30014)\n
---> System.Net.Sockets.SocketException (111): Connection refused\n at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.ThrowException(SocketError
error, CancellationToken cancellationToken)\n at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.System.Threading.Tasks.Sources.IValueTaskSource.GetResult(Int16
token)\n at System.Net.Sockets.Socket.<ConnectAsync>g__WaitForConnectWithCancellation|281_0(AwaitableSocketAsyncEventArgs
saea, ValueTask connectTask, CancellationToken cancellationToken)\n at System.Net.Http.HttpConnectionPool.ConnectToTcpHostAsync(String
host, Int32 port, HttpRequestMessage initialRequest, Boolean async, CancellationToken
cancellationToken)\n --- End of inner exception stack trace ---\n at System.Net.Http.HttpConnectionPool.ConnectToTcpHostAsync(String
host, Int32 port, HttpRequestMessage initialRequest, Boolean async, CancellationToken
cancellationToken)\n at System.Net.Http.HttpConnectionPool.ConnectAsync(HttpRequestMessage
request, Boolean async, CancellationToken cancellationToken)\n at System.Net.Http.HttpConnectionPool.CreateHttp11ConnectionAsync(HttpRequestMessage
request, Boolean async, CancellationToken cancellationToken)\n at System.Net.Http.HttpConnectionPool.AddHttp11ConnectionAsync(QueueItem
queueItem)\n at System.Threading.Tasks.TaskCompletionSourceWithCancellation`1.WaitWithCancellationAsync(CancellationToken
cancellationToken)\n at System.Net.Http.HttpConnectionPool.HttpConnectionWaiter`1.WaitForConnectionAsync(Boolean
async, CancellationToken requestCancellationToken)\n at System.Net.Http.HttpConnectionPool.SendWithVersionDetectionAndRetryAsync(HttpRequestMessage
request, Boolean async, Boolean doRequestAuth, CancellationToken cancellationToken)\n at
System.Net.Http.RedirectHandler.SendAsync(HttpRequestMessage request, Boolean
async, CancellationToken cancellationToken)\n at System.Net.Http.HttpClient.GetStreamAsyncCore(HttpRequestMessage
request, CancellationToken cancellationToken)\n --- End of inner exception
stack trace ---\n at System.Threading.Tasks.Task.ThrowIfExceptional(Boolean
includeTaskCanceledExceptions)\n at System.Threading.Tasks.Task.Wait(Int32
millisecondsTimeout, CancellationToken cancellationToken)\n at System.Threading.Tasks.Task.Wait()\n at
Utils.Time.Wait[T](Task`1 task) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
12\n at Core.Http.<>c__DisplayClass15_0.<HttpGetStream>b__0() in /cs-codex-dist-tests/Framework/Core/Http.cs:line
125\n at Utils.Time.Retry[T](Func`1 action, Int32 maxRetries, TimeSpan retryTime,
String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line 144<---\n\n ---
End of inner exception stack trace ---\n at Utils.Time.Retry[T](Func`1 action,
Int32 maxRetries, TimeSpan retryTime, String description) in /cs-codex-dist-tests/Framework/Utils/Time.cs:line
139\n at Core.Http.LockRetry[T](Func`1 operation, String description) in
/cs-codex-dist-tests/Framework/Core/Http.cs:line 212\n at Core.Http.HttpGetStream(String
route) in /cs-codex-dist-tests/Framework/Core/Http.cs:line 120\n at CodexPlugin.CodexAccess.DownloadFile(String
contentId) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexAccess.cs:line
71\n at CodexPlugin.CodexNode.DownloadToFile(String contentId, TrackedFile
file) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
200\n at CodexPlugin.CodexNode.<>c__DisplayClass35_0.<DownloadContent>b__0()
in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line 128\n at
Logging.Stopwatch.Measure(ILog log, String name, Action action, Boolean debug)
in /cs-codex-dist-tests/Framework/Logging/Stopwatch.cs:line 22\n at CodexPlugin.CodexNode.DownloadContent(ContentId
contentId, String fileLabel) in /cs-codex-dist-tests/ProjectPlugins/CodexPlugin/CodexNode.cs:line
128\n at ContinuousTests.Tests.TwoClientTest.<>c__DisplayClass11_0.<DownloadTestFile>b__0()
in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
43\n at ContinuousTests.Tests.TwoClientTest.LogBytesPerMillisecond(Action
action) in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
82\n at ContinuousTests.Tests.TwoClientTest.DownloadTestFile() in /cs-codex-dist-tests/Tests/CodexContinuousTests/Tests/TwoClientTest.cs:line
43\n at InvokeStub_TwoClientTest.DownloadTestFile(Object, Object, IntPtr*)\n at
System.Reflection.MethodInvoker.Invoke(Object obj, IntPtr* args, BindingFlags
invokeAttr)\n at ContinuousTests.SingleTestRun.ThrowFailTest() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
186\n at ContinuousTests.SingleTestRun.RunTestMoments() in /cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line
155\n at ContinuousTests.SingleTestRun.RunTest(Action`1 resultHandler) in
/cs-codex-dist-tests/Tests/CodexContinuousTests/SingleTestRun.cs:line 76","prometheusid":"codexstorage/dist-tests-prometheus:latest","runid":"20240206-093136","status":"Failed","testduration":"0","testframeworkrevision":"c3eeb0b","testid":"EnvVar-TESTID-NotSet","testname":"TwoClientTest","teststart":"2024-02-06T22:52:53.0881883Z","testtype":"continuous-tests"}}'
headers:
X-elastic-product:
- Elasticsearch
content-length:
- '20829'
content-type:
- application/vnd.elasticsearch+json;compatible-with=8
status:
code: 200
message: OK
version: 1

View File

@ -16,7 +16,7 @@ interactions:
uri: http://localhost:9200/continuous-tests-status-*/_search
response:
body:
string: '{"took":13,"timed_out":false,"_shards":{"total":23,"successful":23,"skipped":0,"failed":0},"hits":{"total":{"value":14,"relation":"eq"},"max_score":null,"hits":[{"_index":"continuous-tests-status-2024.02.07","_id":"t5apg40BTe2pYqi1l9b8","_score":null,"_ignored":["message.keyword"],"_source":{"@timestamp":"2024-02-06T23:07:39.6615501Z","avgupload":"297.8KB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
string: '{"took":4,"timed_out":false,"_shards":{"total":24,"successful":24,"skipped":0,"failed":0},"hits":{"total":{"value":14,"relation":"eq"},"max_score":null,"hits":[{"_index":"continuous-tests-status-2024.02.07","_id":"t5apg40BTe2pYqi1l9b8","_score":null,"_ignored":["message.keyword"],"_source":{"@timestamp":"2024-02-06T23:07:39.6615501Z","avgupload":"297.8KB/s","category":"none","codexcontractsid":"codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests","codexid":"untagged
build","codexrevision":"6869475","error":" data/zDvZRwzm5UemKDPMvadCu999HrqUnCJGzvKnsF7eiy2XV3TzoW7V/network''
timed out after 3 tries over 10 mins, 1 secs.","fixturename":"none","gethid":"codexstorage/dist-tests-geth:latest","involvedpods":"<CODEX24>,<BOOTSTRAP2>","message":"System.Exception:
System.TimeoutException: Retry ''HTTP-GET-STREAM: data/zDvZRwzm5UemKDPMvadCu999HrqUnCJGzvKnsF7eiy2XV3TzoW7V/network''

View File

@ -1,12 +1,14 @@
import pytest
from dateutil import parser
from logtools.log.sources.input.elastic_search.elastic_search_log_repo import ElasticSearchLogRepo, Namespace, Pod
from logtools.resource.core import Namespace, Pod
from logtools.resource.elastic_search_log_repo import ElasticSearchLogRepo
# XXX these are not good quality tests as they are overly complex and either tightly coupled to specific data or very
# weak in terms of what they assert. They will be a pain to maintain. Ideally we should build simpler fixtures and
# test smaller bits at a time, but that requires a lot of setup, so for now we go with this.
# test smaller bits at a time, but that requires a lot of setup (e.g. having pre-constructed index data in fixtures),
# so for now we go with this.
@pytest.mark.vcr
def test_should_retrieve_existing_namespaces():
@ -94,7 +96,7 @@ def test_should_retrieve_failing_test_runs_over_a_single_run_id():
runs = list(repo.test_runs('20240206-093136', failed_only=True))
assert len(runs) == 1
assert runs[0].error.strip() == (
assert repo.describe_test_run(runs[0].id).error.strip() == (
"data/zDvZRwzm5UemKDPMvadCu999HrqUnCJGzvKnsF7eiy2XV3TzoW7V/network' timed out after "
"3 tries over 10 mins, 1 secs."
)

74
poetry.lock generated
View File

@ -273,6 +273,30 @@ docs = ["Jinja2 (==2.11.3)", "MarkupSafe (==1.1.1)", "Pygments (==2.8.1)", "alab
qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"]
testing = ["Django", "attrs", "colorama", "docopt", "pytest (<7.0.0)"]
[[package]]
name = "markdown-it-py"
version = "3.0.0"
description = "Python port of markdown-it. Markdown parsing, done right!"
optional = false
python-versions = ">=3.8"
files = [
{file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"},
{file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"},
]
[package.dependencies]
mdurl = ">=0.1,<1.0"
[package.extras]
benchmarking = ["psutil", "pytest", "pytest-benchmark"]
code-style = ["pre-commit (>=3.0,<4.0)"]
compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"]
linkify = ["linkify-it-py (>=1,<3)"]
plugins = ["mdit-py-plugins"]
profiling = ["gprof2dot"]
rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"]
testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"]
[[package]]
name = "matplotlib-inline"
version = "0.1.6"
@ -287,6 +311,17 @@ files = [
[package.dependencies]
traitlets = "*"
[[package]]
name = "mdurl"
version = "0.1.2"
description = "Markdown URL utilities"
optional = false
python-versions = ">=3.7"
files = [
{file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"},
{file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"},
]
[[package]]
name = "multidict"
version = "6.0.4"
@ -492,23 +527,6 @@ files = [
dev = ["pre-commit", "tox"]
testing = ["pytest", "pytest-benchmark"]
[[package]]
name = "prettytable"
version = "3.9.0"
description = "A simple Python library for easily displaying tabular data in a visually appealing ASCII table format"
optional = false
python-versions = ">=3.8"
files = [
{file = "prettytable-3.9.0-py3-none-any.whl", hash = "sha256:a71292ab7769a5de274b146b276ce938786f56c31cf7cea88b6f3775d82fe8c8"},
{file = "prettytable-3.9.0.tar.gz", hash = "sha256:f4ed94803c23073a90620b201965e5dc0bccf1760b7a7eaf3158cab8aaffdf34"},
]
[package.dependencies]
wcwidth = "*"
[package.extras]
tests = ["pytest", "pytest-cov", "pytest-lazy-fixture"]
[[package]]
name = "prompt-toolkit"
version = "3.0.39"
@ -795,6 +813,24 @@ files = [
{file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"},
]
[[package]]
name = "rich"
version = "13.7.0"
description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal"
optional = false
python-versions = ">=3.7.0"
files = [
{file = "rich-13.7.0-py3-none-any.whl", hash = "sha256:6da14c108c4866ee9520bbffa71f6fe3962e193b7da68720583850cd4548e235"},
{file = "rich-13.7.0.tar.gz", hash = "sha256:5cb5123b5cf9ee70584244246816e9114227e0b98ad9176eede6ad54bf5403fa"},
]
[package.dependencies]
markdown-it-py = ">=2.2.0"
pygments = ">=2.13.0,<3.0.0"
[package.extras]
jupyter = ["ipywidgets (>=7.5.1,<9)"]
[[package]]
name = "six"
version = "1.16.0"
@ -1113,5 +1149,5 @@ multidict = ">=4.0"
[metadata]
lock-version = "2.0"
python-versions = "~3.11"
content-hash = "ca3f368356e3bf637875c8082e3a44b1809a37e4095d2f672deb6f1ee8be54fc"
python-versions = ">=3.11,<3.12"
content-hash = "4362ec2eb2a1d5b64b491a0695ac2aa0509d7779a16fdbfd0e2242a3239497bc"

View File

@ -11,8 +11,8 @@ pytz = "^2023.3.post1"
colored = "^2.2.3"
python-dateutil = "^2.8.2"
elasticsearch = "^8.10.1"
prettytable = "^3.9.0"
fastapi = "^0.109.0"
rich = "^13.7.0"
[tool.poetry.group.dev.dependencies]
pytest = "^7.4.2"