fix log query when no start/end date is provided

logtools/log/sources/input/elastic_search/elastic_search_source.py

@@ -6,6 +6,7 @@ from typing import Optional, Dict, Any, Iterator, Set
 from elasticsearch import Elasticsearch
 
 from logtools.log.base import TimestampedLogLine, LogSource
+from logtools.log.utils import tree
 
 logger = logging.getLogger(__name__)
 
@@ -48,7 +49,8 @@ class ElasticSearchSource(LogSource[TimestampedLogLine[ElasticSearchLocation]]):
     def _indices(self) -> Iterator[str]:
         # FIXME this is a VERY INEFFICIENT fallback
         if self.start_date is None:
-            return [f'{INDEX_PREFIX}-*']
+            yield f'{INDEX_PREFIX}-*'
+            return
 
         start_day = self.start_date.date()
         end_day = self.end_date.date() if self.end_date else datetime.now().date()
@@ -61,9 +63,8 @@ class ElasticSearchSource(LogSource[TimestampedLogLine[ElasticSearchLocation]]):
             start_day += increment
 
     def _build_query(self) -> Dict[str, Any]:
-        query: Dict[str, Any] = {
+        query = tree()
-            'sort': [{'@timestamp': 'asc'}]
+        query['sort'] = [{'@timestamp': 'asc'}]
-        }
 
         if self.start_date is not None or self.end_date is not None:
             time_range = {}
@@ -74,10 +75,12 @@ class ElasticSearchSource(LogSource[TimestampedLogLine[ElasticSearchLocation]]):
             query['query'] = {'bool': {'filter': [{'range': {'@timestamp': time_range}}]}}
 
         if self.pods is not None:
-            query['query']['bool']['filter'].append({"terms": {"pod_name.keyword": list(self.pods)}})
+            filters = query['query']['bool'].setdefault('filter', [])
+            filters.append({"terms": {"pod_name.keyword": list(self.pods)}})
 
         if self.run_id is not None:
-            query['query']['bool']['filter'].append({"term": {"pod_labels.runid.keyword": self.run_id}})
+            filters = query['query']['bool'].setdefault('filter', [])
+            filters.append({"term": {"pod_labels.runid.keyword": self.run_id}})
 
         if 'query' not in query:
             query['query'] = {'match_all': {}}

logtools/log/sources/input/elastic_search/tests/test_elasticsearch_source.py

@@ -27,3 +27,15 @@ def test_should_fetch_logs_by_date():
 
     assert {line.location.pod_name for line in lines} == {'codex1-3-b558568cf-tvcsc', 'bootstrap-2-58b69484bc-88msf'}
     assert {line.location.run_id for line in lines} == {'20231109-101554'}
+
+@pytest.mark.vcr
+def test_should_fetch_logs_when_no_dates_are_specified():
+    log = ElasticSearchSource(
+        run_id='20231109-101554',
+        pods={'codex1-3-b558568cf-tvcsc'}
+    )
+
+    try:
+        next(log.__iter__())
+    except StopIteration:
+        assert False, "Should have returned at least one log line"

logtools/log/tests/test_tree.py

@@ -0,0 +1,16 @@
+from logtools.log.utils import tree
+
+
+def test_should_allow_nodes_to_be_added():
+    a_tree = tree()
+    a_tree['a']['b']['c'] = 'd'
+    a_tree['a']['b']['d'] = 'e'
+
+    assert a_tree == {
+        'a': {
+            'b': {
+                'c': 'd',
+                'd': 'e'
+            }
+        }
+    }

logtools/log/utils.py

@@ -0,0 +1,9 @@
+from collections import defaultdict
+
+from typing import Dict
+
+Tree = Dict[str, 'Tree']
+
+
+def tree() -> Tree:
+    return defaultdict(tree)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "logtools"
-version = "1.0.0"
+version = "1.0.1"
 description = ""
 authors = ["gmega <giuliano@status.im>"]
 readme = "README.md"