Code review: 273060043: Merged skydrive parsers into a single source …

…file log2timeline#388

config/dpkg/changelog

@@ -2,4 +2,4 @@ python-plaso (1.3.1-1) unstable; urgency=low
 
   * Auto-generated
 
- -- Log2Timeline <[email protected]>  Mon, 02 Nov 2015 08:03:19 +0100
+ -- Log2Timeline <[email protected]>  Mon, 02 Nov 2015 21:02:29 +0100

docs/plaso.formatters.rst

@@ -452,14 +452,6 @@ plaso.formatters.skydrivelog module
     :undoc-members:
     :show-inheritance:
 
-plaso.formatters.skydrivelog_old module
----------------------------------------
-
-.. automodule:: plaso.formatters.skydrivelog_old
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 plaso.formatters.skype module
 -----------------------------
 

docs/plaso.parsers.rst

@@ -330,14 +330,6 @@ plaso.parsers.skydrivelog module
     :undoc-members:
     :show-inheritance:
 
-plaso.parsers.skydrivelog_old module
-------------------------------------
-
-.. automodule:: plaso.parsers.skydrivelog_old
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 plaso.parsers.sqlite module
 ---------------------------
 

plaso/formatters/__init__.py

@@ -53,7 +53,6 @@
 from plaso.formatters import selinux
 from plaso.formatters import shell_items
 from plaso.formatters import skydrivelog
-from plaso.formatters import skydrivelog_old
 from plaso.formatters import skype
 from plaso.formatters import symantec
 from plaso.formatters import syslog

plaso/formatters/skydrivelog.py

@@ -22,4 +22,21 @@ class SkyDriveLogFormatter(interface.ConditionalEventFormatter):
   SOURCE_SHORT = u'LOG'
 
 
-manager.FormattersManager.RegisterFormatter(SkyDriveLogFormatter)
+class SkyDriveOldLogFormatter(interface.ConditionalEventFormatter):
+  """Formatter for a SkyDrive old log file event."""
+
+  DATA_TYPE = u'skydrive:log:old:line'
+
+  FORMAT_STRING_PIECES = [
+      u'[{source_code}]',
+      u'({log_level})',
+      u'{text}']
+
+  FORMAT_STRING_SHORT_PIECES = [u'{text}']
+
+  SOURCE_LONG = u'SkyDrive Log File'
+  SOURCE_SHORT = u'LOG'
+
+
+manager.FormattersManager.RegisterFormatters([
+    SkyDriveLogFormatter, SkyDriveOldLogFormatter])

plaso/parsers/__init__.py

@@ -36,7 +36,6 @@
 from plaso.parsers import sccm
 from plaso.parsers import selinux
 from plaso.parsers import skydrivelog
-from plaso.parsers import skydrivelog_old
 from plaso.parsers import sqlite
 from plaso.parsers import symantec
 from plaso.parsers import syslog

plaso/parsers/skydrivelog.py

@@ -20,9 +20,9 @@ class SkyDriveLogEvent(time_events.TimestampEvent):
 
   Attributes:
     detail: The log line details.
+    log_level: Optional, the SkyDrive log level short name.
     module: Optional, the module name that generated the log line.
     source_code: Optional, logging source file and line number.
-    log_level: Optional, the SkyDrive log level short name.
   """
   DATA_TYPE = u'skydrive:log:line'
 
@@ -46,8 +46,36 @@ def __init__(
     self.source_code = source_code
 
 
+class SkyDriveOldLogEvent(time_events.TimestampEvent):
+  """Convenience class for a SkyDrive old log line event.
+
+  Attributes:
+    log_level: The log level used for the event.
+    source_code: Details of the source code file generating the event.
+    text: The log message.
+  """
+  DATA_TYPE = u'skydrive:log:old:line'
+
+  def __init__(self, timestamp, offset, source_code, log_level, text):
+    """Initializes the event object.
+
+    Args:
+      timestamp: The timestamp which is an integer containing the number
+                 of micro seconds since January 1, 1970, 00:00:00 UTC.
+      source_code: Details of the source code file generating the event.
+      log_level: The log level used for the event.
+      text: The log message.
+    """
+    super(SkyDriveOldLogEvent, self).__init__(
+        timestamp, eventdata.EventTimestamp.ADDED_TIME)
+    self.log_level = log_level
+    self.offset = offset
+    self.source_code = source_code
+    self.text = text
+
+
 class SkyDriveLogParser(text_parser.PyparsingMultiLineTextParser):
-  """Parse SkyDrive log files."""
+  """Parses SkyDrive log files."""
 
   NAME = u'skydrive_log'
   DESCRIPTION = u'Parser for OneDrive (or SkyDrive) log files.'
@@ -266,4 +294,176 @@ def VerifyStructure(self, parser_mediator, line):
     return True
 
 
-manager.ParsersManager.RegisterParser(SkyDriveLogParser)
+class SkyDriveOldLogParser(text_parser.PyparsingSingleLineTextParser):
+  """Parse SkyDrive old log files."""
+
+  NAME = u'skydrive_log_old'
+  DESCRIPTION = u'Parser for OneDrive (or SkyDrive) old log files.'
+
+  _ENCODING = u'UTF-8-SIG'
+
+  # Common SDOL (SkyDriveOldLog) pyparsing objects.
+  SDOL_COLON = pyparsing.Literal(u':')
+  SDOL_EXCLAMATION = pyparsing.Literal(u'!')
+
+  # Timestamp (08-01-2013 21:22:28.999).
+  SDOL_TIMESTAMP = (
+      text_parser.PyparsingConstants.DATE_REV +
+      text_parser.PyparsingConstants.TIME_MSEC).setResultsName(
+          u'sdol_timestamp')
+
+  # SkyDrive source code pyparsing structures.
+  SDOL_SOURCE_CODE = pyparsing.Combine(
+      pyparsing.CharsNotIn(u':') +
+      SDOL_COLON +
+      text_parser.PyparsingConstants.INTEGER +
+      SDOL_EXCLAMATION +
+      pyparsing.Word(pyparsing.printables)).setResultsName(u'source_code')
+
+  # SkyDriveOldLogLevel pyparsing structures.
+  SDOL_LOG_LEVEL = (
+      pyparsing.Literal(u'(').suppress() +
+      pyparsing.SkipTo(u')').setResultsName(u'log_level') +
+      pyparsing.Literal(u')').suppress())
+
+  # SkyDrive line pyparsing structure.
+  SDOL_LINE = (
+      SDOL_TIMESTAMP + SDOL_SOURCE_CODE + SDOL_LOG_LEVEL +
+      SDOL_COLON + pyparsing.SkipTo(pyparsing.lineEnd).setResultsName(u'text'))
+
+  # Sometimes the timestamped log line is followed by an empy line,
+  # then by a file name plus other data and finally by another empty
+  # line. It could happen that a logline is split in two parts.
+  # These lines will not be discarded and an event will be generated
+  # ad-hoc (see source), based on the last one if available.
+  SDOL_NO_HEADER_SINGLE_LINE = (
+      pyparsing.Optional(pyparsing.Literal(u'->').suppress()) +
+      pyparsing.SkipTo(pyparsing.lineEnd).setResultsName(u'text'))
+
+  # Define the available log line structures.
+  LINE_STRUCTURES = [
+      (u'logline', SDOL_LINE),
+      (u'no_header_single_line', SDOL_NO_HEADER_SINGLE_LINE),
+  ]
+
+  def __init__(self):
+    """Initializes a parser object."""
+    super(SkyDriveOldLogParser, self).__init__()
+    self.last_event_object = None
+    self.offset = 0
+
+  def _GetTimestamp(self, sdol_timestamp):
+    """Gets a timestamp from a pyparsing ParseResults SkyDriveOld timestamp.
+
+    This is a sdol_timestamp object as returned by using
+    text_parser.PyparsingConstants structures:
+    [[month, day, year], [hours, minutes, seconds], milliseconds], for example
+    [[8, 1, 2013], [21, 22, 28], 999].
+
+    Args:
+      sdol_timestamp: The pyparsing ParseResults object.
+
+    Returns:
+      timestamp: A plaso timelib timestamp or None.
+    """
+    try:
+      month, day, year = sdol_timestamp[0]
+      hour, minute, second = sdol_timestamp[1]
+      millisecond = sdol_timestamp[2]
+      return timelib.Timestamp.FromTimeParts(
+          year, month, day, hour, minute, second,
+          microseconds=millisecond * 1000)
+    except ValueError:
+      pass
+
+  def _ParseLogline(self, structure):
+    """Parse a logline and store appropriate attributes.
+
+    Args:
+      structure: A pyparsing.ParseResults object from a line in the log file.
+
+    Returns:
+      An event object (instance of SkyDriveOldLogEvent) or None.
+    """
+    timestamp = self._GetTimestamp(structure.sdol_timestamp)
+    if not timestamp:
+      logging.debug(u'Invalid timestamp {0:s}'.format(
+          structure.sdol_timestamp))
+      return
+
+    event_object = SkyDriveOldLogEvent(
+        timestamp, self.offset, structure.source_code, structure.log_level,
+        structure.text)
+    self.last_event_object = event_object
+    return event_object
+
+  def _ParseNoHeaderSingleLine(self, structure):
+    """Parse an isolated header line and store appropriate attributes.
+
+    Args:
+      structure: A pyparsing.ParseResults object from an header line in the
+                 log file.
+
+    Returns:
+      An event object (instance of SkyDriveOldLogEvent) or None.
+    """
+    if not self.last_event_object:
+      logging.debug(u'SkyDrive, found isolated line with no previous events')
+      return
+
+    event_object = SkyDriveOldLogEvent(
+        self.last_event_object.timestamp, self.last_event_object.offset, None,
+        None, structure.text)
+    # TODO think to a possible refactoring for the non-header lines.
+    self.last_event_object = None
+    return event_object
+
+  def ParseRecord(self, parser_mediator, key, structure):
+    """Parse each record structure and return an EventObject if applicable.
+
+    Args:
+      parser_mediator: A parser mediator object (instance of ParserMediator).
+      key: An identification string indicating the name of the parsed
+           structure.
+      structure: A pyparsing.ParseResults object from a line in the
+                 log file.
+
+    Returns:
+      An event object (instance of EventObject) or None.
+    """
+    if key == u'logline':
+      return self._ParseLogline(structure)
+    elif key == u'no_header_single_line':
+      return self._ParseNoHeaderSingleLine(structure)
+    else:
+      logging.warning(
+          u'Unable to parse record, unknown structure: {0:s}'.format(key))
+
+  def VerifyStructure(self, parser_mediator, line):
+    """Verify that this file is a SkyDrive old log file.
+
+    Args:
+      parser_mediator: A parser mediator object (instance of ParserMediator).
+      line: A single line from the text file.
+
+    Returns:
+      True if this is the correct parser, False otherwise.
+    """
+    try:
+      parsed_structure = self.SDOL_LINE.parseString(line)
+    except pyparsing.ParseException:
+      logging.debug(u'Not a SkyDrive old log file')
+      return False
+
+    timestamp = self._GetTimestamp(parsed_structure.sdol_timestamp)
+    if not timestamp:
+      logging.debug(
+          u'Not a SkyDrive old log file, invalid timestamp {0:s}'.format(
+              parsed_structure.sdol_timestamp))
+      return False
+
+    return True
+
+
+manager.ParsersManager.RegisterParsers([
+    SkyDriveLogParser, SkyDriveOldLogParser])