From 8eb32672588999b9cc93e249dca3423f36153398 Mon Sep 17 00:00:00 2001 From: Joachim Metz Date: Sun, 31 Mar 2024 17:06:30 +0200 Subject: [PATCH] Changes to date-less log helper #4697 (#4859) --- plaso/containers/events.py | 12 +++--- plaso/engine/timeliner.py | 2 +- plaso/lib/dateless_helper.py | 53 ++++++++++++++++++-------- plaso/parsers/text_plugins/xchatlog.py | 12 +++++- tests/lib/dateless_helper.py | 42 +++++++------------- 5 files changed, 68 insertions(+), 53 deletions(-) diff --git a/plaso/containers/events.py b/plaso/containers/events.py index 51c89643d0..c8c1c5592e 100644 --- a/plaso/containers/events.py +++ b/plaso/containers/events.py @@ -112,14 +112,14 @@ class DateLessLogHelper(interface.AttributeContainer): GRANULARITY_NO_DATE = 'd' # The date-less log format only supports month and day of month. - GRANULARITY_NO_YEARS = 'y' + GRANULARITY_NO_YEAR = 'y' def __init__(self): """Initializes a date-less log helper attribute container.""" super(DateLessLogHelper, self).__init__() self._event_data_stream_identifier = None self.earliest_date = None - self.granularity = self.GRANULARITY_NO_YEARS + self.granularity = self.GRANULARITY_NO_YEAR self.last_relative_date = None self.latest_date = None @@ -132,7 +132,7 @@ def CopyFromYearLessLogHelper(self, year_less_log_helper): year_less_log_helper (YearLessLogHelper): year-less log helper. """ self.earliest_date = (year_less_log_helper.earliest_year, 1, 1) - self.granularity = self.GRANULARITY_NO_YEARS + self.granularity = self.GRANULARITY_NO_YEAR self.last_relative_date = (year_less_log_helper.last_relative_year, 0, 0) self.latest_date = (year_less_log_helper.latest_year, 1, 1) @@ -143,7 +143,7 @@ def GetEarliestDate(self): tuple[int, int, int]: earliest date as tuple of year, month and day of month or None if not available. """ - if self.earliest_date and self.granularity == self.GRANULARITY_NO_YEARS: + if self.earliest_date and self.granularity == self.GRANULARITY_NO_YEAR: return self.earliest_date[0], 0, 0 return self.earliest_date @@ -167,7 +167,7 @@ def GetLastRelativeDate(self): of month or None if not available. """ if (self.last_relative_date and - self.granularity == self.GRANULARITY_NO_YEARS): + self.granularity == self.GRANULARITY_NO_YEAR): return self.last_relative_date[0], 0, 0 return self.last_relative_date @@ -179,7 +179,7 @@ def GetLatestDate(self): tuple[int, int, int]: latest date as tuple of year, month and day of month or None if not available. """ - if self.latest_date and self.granularity == self.GRANULARITY_NO_YEARS: + if self.latest_date and self.granularity == self.GRANULARITY_NO_YEAR: return self.latest_date[0], 0, 0 return self.latest_date diff --git a/plaso/engine/timeliner.py b/plaso/engine/timeliner.py index 88758153f5..fd5869a21f 100644 --- a/plaso/engine/timeliner.py +++ b/plaso/engine/timeliner.py @@ -124,7 +124,7 @@ def _GetBaseDate(self, storage_writer, event_data): latest_date = date_less_log_helper.GetLatestDate() if date_less_log_helper.granularity == ( - date_less_log_helper.GRANULARITY_NO_YEARS): + date_less_log_helper.GRANULARITY_NO_YEAR): current_date = (current_date[0], 0, 0) if earliest_date is None or last_relative_date is None: diff --git a/plaso/lib/dateless_helper.py b/plaso/lib/dateless_helper.py index a7d82b1af7..bbb2a0918a 100644 --- a/plaso/lib/dateless_helper.py +++ b/plaso/lib/dateless_helper.py @@ -26,14 +26,20 @@ class DateLessLogFormatHelper(object): _VALID_MONTHS = frozenset(range(1, 13)) + # The date-less log format only supports time. + _GRANULARITY_NO_DATE = 'd' + + # The date-less log format only supports month and day of month. + _GRANULARITY_NO_YEAR = 'y' + def __init__(self): """Initializes the date-less log format helper mix-in.""" super(DateLessLogFormatHelper, self).__init__() self._base_date = None + self._date = (0, 0, 0) + self._granularity = self._GRANULARITY_NO_YEAR self._maximum_date = None - self._month = None self._relative_date = (0, 0, 0) - self._year = 0 def _GetDatesFromFileEntry(self, file_entry): """Retrieves the dates from the file entry date and time values. @@ -93,20 +99,20 @@ def _GetYear(self): Returns: int: year. """ - return self._year + return self._date[0] - def _SetEstimatedYear(self, parser_mediator): - """Sets the year based on the parser mediator year estimation. + def _SetEstimatedDate(self, parser_mediator): + """Estimate the date based on the file entry dates. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfVFS. """ self._base_date = None + self._date = (0, 0, 0) + self._granularity = self._GRANULARITY_NO_DATE self._maximum_date = None - self._month = None self._relative_date = (0, 0, 0) - self._year = 0 dates = set() @@ -126,8 +132,20 @@ def _SetEstimatedYear(self, parser_mediator): if dates: self._base_date = min(dates) + self._date = self._base_date self._maximum_date = max(dates) - self._year = self._base_date[0] + + def _SetEstimatedYear(self, parser_mediator): + """Estimate the year based on the file entry dates. + + Args: + parser_mediator (ParserMediator): mediates interactions between parsers + and other components, such as storage and dfVFS. + """ + self._SetEstimatedDate(parser_mediator) + + self._date = (self._date[0], 0, 0) + self._granularity = self._GRANULARITY_NO_YEAR def _SetMonthAndYear(self, month, year): """Sets the month and year. @@ -142,9 +160,9 @@ def _SetMonthAndYear(self, month, year): if month not in self._VALID_MONTHS: raise ValueError('Invalid month: {0!s}'.format(month)) - self._month = month + self._date = (year, month, 0) + self._granularity = self._GRANULARITY_NO_YEAR self._relative_date = (0, 0, 0) - self._year = year def _UpdateYear(self, month): """Updates the year based on the month observed in the log format. @@ -158,25 +176,27 @@ def _UpdateYear(self, month): if month not in self._VALID_MONTHS: raise ValueError('Invalid month: {0!s}'.format(month)) - if self._month: + last_year, last_month, _ = self._date + + if last_month: relative_year, relative_month, relative_day_of_month = self._relative_date # Account for log formats that allow out-of-order date and time values # (Apr->May->Apr) such as rsyslog with the RepeatedMsgReduction setting # enabled. - if month + 1 < self._month: + if month + 1 < last_month: self._relative_date = ( relative_year + 1, relative_month, relative_day_of_month) - self._year += 1 + last_year += 1 # Account for out-of-order Jan->Dec->Jan with the exception of the start # of the log file. - elif relative_year > 0 and self._month == 1 and month == 12: + elif relative_year > 0 and last_month == 1 and month == 12: self._relative_date = ( relative_year - 1, relative_month, relative_day_of_month) - self._year -= 1 + last_year -= 1 - self._month = month + self._date = (last_year, month, 0) def GetDateLessLogHelper(self): """Retrieves a date-less log helper attribute container. @@ -186,6 +206,7 @@ def GetDateLessLogHelper(self): """ date_less_log_helper = events.DateLessLogHelper() date_less_log_helper.earliest_date = self._base_date + date_less_log_helper.granularity = self._granularity date_less_log_helper.last_relative_date = self._relative_date date_less_log_helper.latest_date = self._maximum_date diff --git a/plaso/parsers/text_plugins/xchatlog.py b/plaso/parsers/text_plugins/xchatlog.py index 24378ca15e..2af5ce8339 100644 --- a/plaso/parsers/text_plugins/xchatlog.py +++ b/plaso/parsers/text_plugins/xchatlog.py @@ -161,6 +161,11 @@ class XChatLogTextPlugin( VERIFICATION_GRAMMAR = _SECTION_HEADER_LINE + def __init__(self): + """Initializes a text parser plugin.""" + super(XChatLogTextPlugin, self).__init__() + self._year = None + def _ParseLogLine(self, parser_mediator, structure): """Parses a log line. @@ -222,8 +227,9 @@ def _ParseSectionHeaderLine(self, parser_mediator, structure): structure, 'log_action', default_value=[]) if log_action[0] not in ('BEGIN', 'END'): + log_action_string = ' '.join(log_action) parser_mediator.ProduceExtractionWarning( - 'unsupported log action: {0:s}.'.format(' '.join(log_action))) + f'unsupported log action: {log_action_string:s}') self._year = None return @@ -272,6 +278,8 @@ def _ParseTimeElements(self, time_elements_structure): self._SetMonthAndYear(month, year) + self._year = year + time_elements_tuple = (year, month, day_of_month, hours, minutes, seconds) date_time = dfdatetime_time_elements.TimeElements( @@ -283,7 +291,7 @@ def _ParseTimeElements(self, time_elements_structure): except (TypeError, ValueError) as exception: raise errors.ParseError( - 'Unable to parse time elements with error: {0!s}'.format(exception)) + f'Unable to parse time elements with error: {exception!s}') def CheckRequiredFormat(self, parser_mediator, text_reader): """Check if the log record has the minimal structure required by the plugin. diff --git a/tests/lib/dateless_helper.py b/tests/lib/dateless_helper.py index ca05d6c5f3..e4e306d927 100644 --- a/tests/lib/dateless_helper.py +++ b/tests/lib/dateless_helper.py @@ -75,15 +75,13 @@ def testSetEstimatedYear(self): test_helper = dateless_helper.DateLessLogFormatHelper() + self.assertEqual(test_helper._date, (0, 0, 0)) self.assertEqual(test_helper._relative_date, (0, 0, 0)) - self.assertEqual(test_helper._year, 0) - self.assertIsNone(test_helper._month) test_helper._SetEstimatedYear(parser_mediator) + self.assertEqual(test_helper._date, (2012, 0, 0)) self.assertEqual(test_helper._relative_date, (0, 0, 0)) - self.assertEqual(test_helper._year, 2012) - self.assertIsNone(test_helper._month) test_path = self._GetTestFilePath(['syslog.xz']) os_path_spec = path_spec_factory.Factory.NewPathSpec( @@ -102,81 +100,69 @@ def testSetEstimatedYear(self): test_helper = dateless_helper.DateLessLogFormatHelper() + self.assertEqual(test_helper._date, (0, 0, 0)) self.assertEqual(test_helper._relative_date, (0, 0, 0)) - self.assertEqual(test_helper._year, 0) - self.assertIsNone(test_helper._month) test_helper._SetEstimatedYear(parser_mediator) expected_date = min(dates) + self.assertEqual(test_helper._date, (expected_date[0], 0, 0)) self.assertEqual(test_helper._relative_date, (0, 0, 0)) - self.assertEqual(test_helper._year, expected_date[0]) - self.assertIsNone(test_helper._month) def testSetMonthAndYear(self): """Tests the _SetMonthAndYear function.""" test_helper = dateless_helper.DateLessLogFormatHelper() + self.assertEqual(test_helper._date, (0, 0, 0)) self.assertEqual(test_helper._relative_date, (0, 0, 0)) - self.assertEqual(test_helper._year, 0) - self.assertIsNone(test_helper._month) test_helper._SetMonthAndYear(11, 2022) + self.assertEqual(test_helper._date, (2022, 11, 0)) self.assertEqual(test_helper._relative_date, (0, 0, 0)) - self.assertEqual(test_helper._year, 2022) - self.assertEqual(test_helper._month, 11) def testUpdateYear(self): """Tests the _UpdateYear function.""" test_helper = dateless_helper.DateLessLogFormatHelper() + self.assertEqual(test_helper._date, (0, 0, 0)) self.assertEqual(test_helper._relative_date, (0, 0, 0)) - self.assertEqual(test_helper._year, 0) - self.assertIsNone(test_helper._month) test_helper._UpdateYear(1) + self.assertEqual(test_helper._date, (0, 1, 0)) self.assertEqual(test_helper._relative_date, (0, 0, 0)) - self.assertEqual(test_helper._year, 0) - self.assertEqual(test_helper._month, 1) test_helper._UpdateYear(5) + self.assertEqual(test_helper._date, (0, 5, 0)) self.assertEqual(test_helper._relative_date, (0, 0, 0)) - self.assertEqual(test_helper._year, 0) - self.assertEqual(test_helper._month, 5) test_helper._UpdateYear(12) + self.assertEqual(test_helper._date, (0, 12, 0)) self.assertEqual(test_helper._relative_date, (0, 0, 0)) - self.assertEqual(test_helper._year, 0) - self.assertEqual(test_helper._month, 12) test_helper._UpdateYear(1) + self.assertEqual(test_helper._date, (1, 1, 0)) self.assertEqual(test_helper._relative_date, (1, 0, 0)) - self.assertEqual(test_helper._year, 1) - self.assertEqual(test_helper._month, 1) test_helper._UpdateYear(12) + self.assertEqual(test_helper._date, (0, 12, 0)) self.assertEqual(test_helper._relative_date, (0, 0, 0)) - self.assertEqual(test_helper._year, 0) - self.assertEqual(test_helper._month, 12) test_helper._UpdateYear(5) + self.assertEqual(test_helper._date, (1, 5, 0)) self.assertEqual(test_helper._relative_date, (1, 0, 0)) - self.assertEqual(test_helper._year, 1) - self.assertEqual(test_helper._month, 5) test_helper._UpdateYear(1) + self.assertEqual(test_helper._date, (2, 1, 0)) self.assertEqual(test_helper._relative_date, (2, 0, 0)) - self.assertEqual(test_helper._year, 2) - self.assertEqual(test_helper._month, 1) def testGetDateLessLogHelper(self): """Tests the GetDateLessLogHelper function."""