From 5b77c92b62bee8cce40973361125b070cec7f85d Mon Sep 17 00:00:00 2001 From: Roger Date: Tue, 31 Jan 2023 12:53:41 +0100 Subject: [PATCH 1/3] fix scan timeframe with use_terms_query --- elastalert/elastalert.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 65d0ce7b..e3478105 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -698,7 +698,8 @@ def set_starttime(self, rule, endtime): # Query from the end of the last run, if it exists, otherwise a run_every sized window rule['starttime'] = rule.get('previous_endtime', endtime - self.run_every) else: - rule['starttime'] = rule.get('previous_endtime', endtime - rule['timeframe']) + #Based on PR 3141 old Yelp/elastalert - rschirin + rule['starttime'] = endtime - rule['timeframe'] def adjust_start_time_for_overlapping_agg_query(self, rule): if rule.get('aggregation_query_element'): From e532e9e7dbb12ebd9042fb7150d1dbe4b614b361 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 1 Feb 2023 12:06:13 -0500 Subject: [PATCH 2/3] add missing unit test and changelog --- CHANGELOG.md | 1 + tests/base_test.py | 24 ++++++++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 643d534b..3c100957 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - Add support for Kibana 8.6 for Kibana Discover - [#1080](https://github.com/jertel/elastalert2/pull/1080) - @nsano-rururu - Modify schema to allow string and boolean for `*_ca_certs` to allow for one to specify a cert bundle for SSL certificate verification - [#1082](https://github.com/jertel/elastalert2/pull/1082) - @goggin - Fix UnicodeEncodeError in PagerDutyAlerter - [#1091](https://github.com/jertel/elastalert2/pull/1091) - @nsano-rururu +- The scan_entire_timeframe setting, when used with use_count_query or use_terms_query will not scan entire timeframe on subsequent rule runs - [#1097](https://github.com/jertel/elastalert2/pull/1097) - @rschirin # 2.9.0 diff --git a/tests/base_test.py b/tests/base_test.py index 5be65ee8..450b593b 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -876,8 +876,7 @@ def test_set_starttime(ea): ea.set_starttime(ea.rules[0], end) assert ea.rules[0]['starttime'] == end - ea.buffer_time - # scan_entire_timeframe - ea.rules[0].pop('previous_endtime') + # scan_entire_timeframe without use_count_query or use_terms_query ea.rules[0].pop('starttime') ea.rules[0]['timeframe'] = datetime.timedelta(days=3) ea.rules[0]['scan_entire_timeframe'] = True @@ -886,6 +885,27 @@ def test_set_starttime(ea): ea.set_starttime(ea.rules[0], end) assert ea.rules[0]['starttime'] == end - datetime.timedelta(days=3) + # scan_entire_timeframe with use_count_query, first run + ea.rules[0].pop('starttime') + ea.rules[0]['timeframe'] = datetime.timedelta(days=3) + ea.rules[0]['scan_entire_timeframe'] = True + ea.rules[0]['use_count_query'] = True + with mock.patch.object(ea, 'get_starttime') as mock_gs: + mock_gs.return_value = None + ea.set_starttime(ea.rules[0], end) + assert ea.rules[0]['starttime'] == end - datetime.timedelta(days=3) + + # scan_entire_timeframe with use_count_query, subsequent run + ea.rules[0].pop('starttime') + ea.rules[0]['timeframe'] = datetime.timedelta(days=3) + ea.rules[0]['scan_entire_timeframe'] = True + ea.rules[0]['use_count_query'] = True + ea.rules[0]['previous_endtime'] = end + with mock.patch.object(ea, 'get_starttime') as mock_gs: + mock_gs.return_value = None + ea.set_starttime(ea.rules[0], end) + assert ea.rules[0]['starttime'] == end - datetime.timedelta(days=3) + def test_rule_changes(ea): ea.rule_hashes = {'rules/rule1.yaml': 'ABC', From 295ca062eacf456a97e2569baa2dbd94d4f5c7fe Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 1 Feb 2023 12:16:48 -0500 Subject: [PATCH 3/3] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c100957..f447eb5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ - Add support for Kibana 8.6 for Kibana Discover - [#1080](https://github.com/jertel/elastalert2/pull/1080) - @nsano-rururu - Modify schema to allow string and boolean for `*_ca_certs` to allow for one to specify a cert bundle for SSL certificate verification - [#1082](https://github.com/jertel/elastalert2/pull/1082) - @goggin - Fix UnicodeEncodeError in PagerDutyAlerter - [#1091](https://github.com/jertel/elastalert2/pull/1091) - @nsano-rururu -- The scan_entire_timeframe setting, when used with use_count_query or use_terms_query will not scan entire timeframe on subsequent rule runs - [#1097](https://github.com/jertel/elastalert2/pull/1097) - @rschirin +- The scan_entire_timeframe setting, when used with use_count_query or use_terms_query will now scan entire timeframe on subsequent rule runs - [#1097](https://github.com/jertel/elastalert2/pull/1097) - @rschirin # 2.9.0