fix log query when no start/end date is provided
This commit is contained in:
parent
befc080850
commit
09075b9833
|
@ -6,6 +6,7 @@ from typing import Optional, Dict, Any, Iterator, Set
|
|||
from elasticsearch import Elasticsearch
|
||||
|
||||
from logtools.log.base import TimestampedLogLine, LogSource
|
||||
from logtools.log.utils import tree
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
@ -48,7 +49,8 @@ class ElasticSearchSource(LogSource[TimestampedLogLine[ElasticSearchLocation]]):
|
|||
def _indices(self) -> Iterator[str]:
|
||||
# FIXME this is a VERY INEFFICIENT fallback
|
||||
if self.start_date is None:
|
||||
return [f'{INDEX_PREFIX}-*']
|
||||
yield f'{INDEX_PREFIX}-*'
|
||||
return
|
||||
|
||||
start_day = self.start_date.date()
|
||||
end_day = self.end_date.date() if self.end_date else datetime.now().date()
|
||||
|
@ -61,9 +63,8 @@ class ElasticSearchSource(LogSource[TimestampedLogLine[ElasticSearchLocation]]):
|
|||
start_day += increment
|
||||
|
||||
def _build_query(self) -> Dict[str, Any]:
|
||||
query: Dict[str, Any] = {
|
||||
'sort': [{'@timestamp': 'asc'}]
|
||||
}
|
||||
query = tree()
|
||||
query['sort'] = [{'@timestamp': 'asc'}]
|
||||
|
||||
if self.start_date is not None or self.end_date is not None:
|
||||
time_range = {}
|
||||
|
@ -74,10 +75,12 @@ class ElasticSearchSource(LogSource[TimestampedLogLine[ElasticSearchLocation]]):
|
|||
query['query'] = {'bool': {'filter': [{'range': {'@timestamp': time_range}}]}}
|
||||
|
||||
if self.pods is not None:
|
||||
query['query']['bool']['filter'].append({"terms": {"pod_name.keyword": list(self.pods)}})
|
||||
filters = query['query']['bool'].setdefault('filter', [])
|
||||
filters.append({"terms": {"pod_name.keyword": list(self.pods)}})
|
||||
|
||||
if self.run_id is not None:
|
||||
query['query']['bool']['filter'].append({"term": {"pod_labels.runid.keyword": self.run_id}})
|
||||
filters = query['query']['bool'].setdefault('filter', [])
|
||||
filters.append({"term": {"pod_labels.runid.keyword": self.run_id}})
|
||||
|
||||
if 'query' not in query:
|
||||
query['query'] = {'match_all': {}}
|
||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -27,3 +27,15 @@ def test_should_fetch_logs_by_date():
|
|||
|
||||
assert {line.location.pod_name for line in lines} == {'codex1-3-b558568cf-tvcsc', 'bootstrap-2-58b69484bc-88msf'}
|
||||
assert {line.location.run_id for line in lines} == {'20231109-101554'}
|
||||
|
||||
@pytest.mark.vcr
|
||||
def test_should_fetch_logs_when_no_dates_are_specified():
|
||||
log = ElasticSearchSource(
|
||||
run_id='20231109-101554',
|
||||
pods={'codex1-3-b558568cf-tvcsc'}
|
||||
)
|
||||
|
||||
try:
|
||||
next(log.__iter__())
|
||||
except StopIteration:
|
||||
assert False, "Should have returned at least one log line"
|
||||
|
|
|
@ -0,0 +1,16 @@
|
|||
from logtools.log.utils import tree
|
||||
|
||||
|
||||
def test_should_allow_nodes_to_be_added():
|
||||
a_tree = tree()
|
||||
a_tree['a']['b']['c'] = 'd'
|
||||
a_tree['a']['b']['d'] = 'e'
|
||||
|
||||
assert a_tree == {
|
||||
'a': {
|
||||
'b': {
|
||||
'c': 'd',
|
||||
'd': 'e'
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,9 @@
|
|||
from collections import defaultdict
|
||||
|
||||
from typing import Dict
|
||||
|
||||
Tree = Dict[str, 'Tree']
|
||||
|
||||
|
||||
def tree() -> Tree:
|
||||
return defaultdict(tree)
|
|
@ -1,6 +1,6 @@
|
|||
[tool.poetry]
|
||||
name = "logtools"
|
||||
version = "1.0.0"
|
||||
version = "1.0.1"
|
||||
description = ""
|
||||
authors = ["gmega <giuliano@status.im>"]
|
||||
readme = "README.md"
|
||||
|
|
Loading…
Reference in New Issue