Code review: 264250043: Initial version of $MFT parser log2timeline#316

ACKNOWLEDGEMENTS

@@ -140,6 +140,7 @@ Copyright SANS Institute - Digital Forensics and Incident Response.
 * 5afe4de1b92fc382.customDestinations-ms
 * Catalog1.edb
 * example.lnk
+* MFT
 * nfury_index.dat
 * Ntuser.dat (multiple instances)
 * SysEvent.Evt

config/dpkg/changelog

@@ -2,4 +2,4 @@ python-plaso (1.3.1-1) unstable; urgency=low
 
   * Auto-generated
 
- -- Log2Timeline <[email protected]>  Tue, 29 Sep 2015 21:29:06 +0200
+ -- Log2Timeline <[email protected]>  Tue, 29 Sep 2015 22:18:57 +0200

docs/plaso.events.rst

@@ -4,6 +4,14 @@ plaso.events package
 Submodules
 ----------
 
+plaso.events.file_system_events module
+--------------------------------------
+
+.. automodule:: plaso.events.file_system_events
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 plaso.events.plist_event module
 -------------------------------
 

docs/plaso.formatters.rst

@@ -140,10 +140,10 @@ plaso.formatters.file_history module
     :undoc-members:
     :show-inheritance:
 
-plaso.formatters.filestat module
---------------------------------
+plaso.formatters.file_system module
+-----------------------------------
 
-.. automodule:: plaso.formatters.filestat
+.. automodule:: plaso.formatters.file_system
     :members:
     :undoc-members:
     :show-inheritance:

docs/plaso.parsers.rst

@@ -210,6 +210,14 @@ plaso.parsers.msiecf module
     :undoc-members:
     :show-inheritance:
 
+plaso.parsers.ntfs module
+-------------------------
+
+.. automodule:: plaso.parsers.ntfs
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 plaso.parsers.olecf module
 --------------------------
 

plaso/dependencies.py

@@ -16,7 +16,7 @@
     u'pyevt': 20120410,
     u'pyevtx': 20141112,
     u'pyewf': 20131210,
-    u'pyfsntfs': 20150829,
+    u'pyfsntfs': 20150831,
     u'pyfwsi': 20150606,
     u'pylnk': 20150830,
     u'pymsiecf': 20150314,
@@ -39,7 +39,7 @@
     (u'binplist', u'__version__', u'0.1.4', None),
     (u'construct', u'__version__', u'2.5.2', None),
     (u'dateutil', u'__version__', u'1.5', None),
-    (u'dfvfs', u'__version__', u'20150829', None),
+    (u'dfvfs', u'__version__', u'20150915', None),
     (u'dpkt', u'__version__', u'1.8', None),
     # The protobuf module does not appear to have version information.
     (u'google.protobuf', u'', u'', None),

plaso/events/file_system_events.py

@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+"""This file contains the file system specific event object classes."""
+
+from plaso.events import time_events
+
+
+class FileStatEvent(time_events.TimestampEvent):
+  """File system stat event.
+
+  Attributes:
+    file_size: the file size.
+    file_system_type: the file system type.
+    is_allocated: boolean value to indicate the file is allocated.
+    offset: the offset of the stat data.
+  """
+
+  DATA_TYPE = u'fs:stat'
+
+  def __init__(
+      self, timestamp, timestamp_description, is_allocated, file_size,
+      file_system_type):
+    """Initializes the event object.
+
+    Args:
+      timestamp: the timestamp time value. The timestamp contains the
+                 number of microseconds since Jan 1, 1970 00:00:00 UTC
+      timestamp_description: a description string for the timestamp value.
+      is_allocated: boolean value to indicate the file entry is allocated.
+      file_size: an integer containing the file size in bytes.
+      file_system_type: a string containing the file system type.
+    """
+    super(FileStatEvent, self).__init__(timestamp, timestamp_description)
+
+    self.file_size = file_size
+    self.file_system_type = file_system_type
+    self.is_allocated = is_allocated
+    self.offset = 0
+
+
+class NTFSFileStatEvent(time_events.FiletimeEvent):
+  """NTFS file system stat event.
+
+  Attributes:
+    attribute_type: the attribute type e.g. 0x00000030 which represents
+                    $FILE_NAME.
+    file_attribute_flags: the NTFS file attribute flags, set to None
+                          if not available.
+    file_reference: NTFS file reference.
+    file_system_type: the file system type.
+    is_allocated: boolean value to indicate the MFT entry is allocated
+                 (marked as in use).
+    name: string containing the name associated with the stat event, e.g.
+          that of a $FILE_NAME attribute, set to None if not available.
+    offset: the offset of the stat data.
+    parent_file_reference: NTFS file reference of the parent, set to None
+                           if not available.
+  """
+
+  DATA_TYPE = u'fs:stat:ntfs'
+
+  def __init__(
+      self, timestamp, timestamp_description, file_reference, attribute_type,
+      file_attribute_flags=None, is_allocated=True, name=None,
+      parent_file_reference=None):
+    """Initializes the event object.
+
+    Args:
+      timestamp: the FILETIME value for the timestamp.
+      timestamp_description: the usage string for the timestamp value.
+      file_reference: NTFS file reference.
+      attribute_type: the attribute type e.g. 0x00000030 which represents
+                      $FILE_NAME.
+      file_attribute_flags: optional NTFS file attribute flags, set to None
+                            if not available.
+      is_allocated: optional boolean value to indicate the MFT entry is
+                    is allocated (marked as in use).
+      name: optional string containing the name associated with the stat event,
+            e.g.  that of a $FILE_NAME attribute, set to None if not available.
+      parent_file_reference: optional NTFS file reference of the parent, set
+                             to None if not available.
+    """
+    super(NTFSFileStatEvent, self).__init__(timestamp, timestamp_description)
+
+    self.attribute_type = attribute_type
+    self.file_reference = file_reference
+    self.file_attribute_flags = file_attribute_flags
+    self.file_system_type = u'NTFS'
+    self.is_allocated = is_allocated
+    self.name = name
+    self.offset = 0
+    self.parent_file_reference = parent_file_reference

plaso/formatters/__init__.py

@@ -16,8 +16,8 @@
 from plaso.formatters import chrome_extension_activity
 from plaso.formatters import chrome_preferences
 from plaso.formatters import cups_ipp
-from plaso.formatters import filestat
 from plaso.formatters import file_history
+from plaso.formatters import file_system
 from plaso.formatters import firefox
 from plaso.formatters import firefox_cache
 from plaso.formatters import firefox_cookies

plaso/formatters/file_system.py

@@ -0,0 +1,139 @@
+# -*- coding: utf-8 -*-
+"""The file system stat event formatter."""
+
+from plaso.formatters import interface
+from plaso.formatters import manager
+from plaso.lib import errors
+
+
+class FileStatEventFormatter(interface.ConditionalEventFormatter):
+  """The file system stat event formatter."""
+
+  DATA_TYPE = u'fs:stat'
+
+  FORMAT_STRING_PIECES = [
+      u'{display_name}',
+      u'({unallocated})']
+
+  FORMAT_STRING_SHORT_PIECES = [
+      u'{filename}']
+
+  SOURCE_SHORT = u'FILE'
+
+  def GetMessages(self, unused_formatter_mediator, event_object):
+    """Determines the formatted message strings for an event object.
+
+    Args:
+      formatter_mediator: the formatter mediator object (instance of
+                          FormatterMediator).
+      event_object: the event object (instance of EventObject).
+
+    Returns:
+      A tuple containing the formatted message string and short message string.
+
+    Raises:
+      WrongFormatter: if the event object cannot be formatted by the formatter.
+    """
+    if self.DATA_TYPE != event_object.data_type:
+      raise errors.WrongFormatter(u'Unsupported data type: {0:s}.'.format(
+          event_object.data_type))
+
+    event_values = event_object.GetValues()
+
+    # The usage of allocated is deprecated in favor of is_allocated but
+    # is kept here to be backwards compatible.
+    if (not event_values.get(u'allocated', False) and
+        not event_values.get(u'is_allocated', False)):
+      event_values[u'unallocated'] = u'unallocated'
+
+    return self._ConditionalFormatMessages(event_values)
+
+  def GetSources(self, event_object):
+    """Determines the the short and long source for an event object.
+
+    Args:
+      event_object: the event object (instance of EventObject).
+
+    Returns:
+      A tuple of the short and long source string.
+
+    Raises:
+      WrongFormatter: if the event object cannot be formatted by the formatter.
+    """
+    if self.DATA_TYPE != event_object.data_type:
+      raise errors.WrongFormatter(u'Unsupported data type: {0:s}.'.format(
+          event_object.data_type))
+
+    file_system_type = getattr(event_object, u'file_system_type', u'UNKNOWN')
+    timestamp_desc = getattr(event_object, u'timestamp_desc', u'Time')
+    source_long = u'{0:s} {1:s}'.format(file_system_type, timestamp_desc)
+
+    return self.SOURCE_SHORT, source_long
+
+
+class NTFSFileStatEventFormatter(FileStatEventFormatter):
+  """The NTFS file system stat event formatter."""
+
+  DATA_TYPE = u'fs:stat:ntfs'
+
+  FORMAT_STRING_PIECES = [
+      u'{display_name}',
+      u'File reference: {file_reference}',
+      u'Attribute name: {attribute_name}',
+      u'Name: {name}',
+      u'Parent file reference: {parent_file_reference}',
+      u'({unallocated})']
+
+  FORMAT_STRING_SHORT_PIECES = [
+      u'{filename}',
+      u'{file_reference}',
+      u'{attribute_name}']
+
+  SOURCE_SHORT = u'FILE'
+
+  _ATTRIBUTE_NAMES = {
+      0x00000010: u'$STANDARD_INFORMATION',
+      0x00000030: u'$FILE_NAME'
+  }
+
+  def GetMessages(self, unused_formatter_mediator, event_object):
+    """Determines the formatted message strings for an event object.
+
+    Args:
+      formatter_mediator: the formatter mediator object (instance of
+                          FormatterMediator).
+      event_object: the event object (instance of EventObject).
+
+    Returns:
+      A tuple containing the formatted message string and short message string.
+
+    Raises:
+      WrongFormatter: if the event object cannot be formatted by the formatter.
+    """
+    if self.DATA_TYPE != event_object.data_type:
+      raise errors.WrongFormatter(u'Unsupported data type: {0:s}.'.format(
+          event_object.data_type))
+
+    event_values = event_object.GetValues()
+
+    attribute_type = event_values.get(u'attribute_type', 0)
+    event_values[u'attribute_name'] = self._ATTRIBUTE_NAMES.get(
+        attribute_type, u'UNKNOWN')
+
+    file_reference = event_values.get(u'file_reference', 0)
+    event_values[u'file_reference'] = u'{0:d}-{1:d}'.format(
+        file_reference & 0xffffffffffff, file_reference >> 48)
+
+    parent_file_reference = event_values.get(u'parent_file_reference', 0)
+    if parent_file_reference:
+      event_values[u'parent_file_reference'] = u'{0:d}-{1:d}'.format(
+          parent_file_reference & 0xffffffffffff, parent_file_reference >> 48)
+
+    if not event_values.get(u'is_allocated', False):
+      event_values[u'unallocated'] = u'unallocated'
+
+    return self._ConditionalFormatMessages(event_values)
+
+
+manager.FormattersManager.RegisterFormatters([
+    FileStatEventFormatter, NTFSFileStatEventFormatter])

plaso/parsers/__init__.py

@@ -22,6 +22,7 @@
 from plaso.parsers import mactime
 from plaso.parsers import mcafeeav
 from plaso.parsers import msiecf
+from plaso.parsers import ntfs
 from plaso.parsers import olecf
 from plaso.parsers import opera
 from plaso.parsers import oxml