Code review: 240160044: Moving output modules to use argument helpers…

…, re-factor of psort into tools/frontend and bugfixes

config/dpkg/changelog

@@ -2,4 +2,4 @@ python-plaso (1.2.1-1) unstable; urgency=low
 
   * Auto-generated
 
- -- Log2Timeline <[email protected]>  Wed, 03 Jun 2015 20:43:58 +0200
+ -- Log2Timeline <[email protected]>  Wed, 03 Jun 2015 12:12:02 -0700

docs/plaso.cli.helpers.rst

@@ -4,6 +4,14 @@ plaso.cli.helpers package
 Submodules
 ----------
 
+plaso.cli.helpers.database_config module
+----------------------------------------
+
+.. automodule:: plaso.cli.helpers.database_config
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 plaso.cli.helpers.elastic_output module
 ---------------------------------------
 
@@ -36,6 +44,46 @@ plaso.cli.helpers.manager_test module
     :undoc-members:
     :show-inheritance:
 
+plaso.cli.helpers.mysql_4n6time_output module
+---------------------------------------------
+
+.. automodule:: plaso.cli.helpers.mysql_4n6time_output
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+plaso.cli.helpers.pstorage module
+---------------------------------
+
+.. automodule:: plaso.cli.helpers.pstorage
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+plaso.cli.helpers.server_config module
+--------------------------------------
+
+.. automodule:: plaso.cli.helpers.server_config
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+plaso.cli.helpers.shared_4n6time_output module
+----------------------------------------------
+
+.. automodule:: plaso.cli.helpers.shared_4n6time_output
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+plaso.cli.helpers.sqlite_4n6time_output module
+----------------------------------------------
+
+.. automodule:: plaso.cli.helpers.sqlite_4n6time_output
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 plaso.cli.helpers.tagging_analysis module
 -----------------------------------------
 
@@ -44,6 +92,14 @@ plaso.cli.helpers.tagging_analysis module
     :undoc-members:
     :show-inheritance:
 
+plaso.cli.helpers.timesketch_out module
+---------------------------------------
+
+.. automodule:: plaso.cli.helpers.timesketch_out
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 plaso.cli.helpers.virustotal_analysis module
 --------------------------------------------
 

plaso/cli/helpers/__init__.py

@@ -2,5 +2,9 @@
 """This file contains an import statement for each argument helper."""
 
 from plaso.cli.helpers import elastic_output
+from plaso.cli.helpers import mysql_4n6time_output
+from plaso.cli.helpers import sqlite_4n6time_output
+from plaso.cli.helpers import pstorage
+from plaso.cli.helpers import timesketch_out
 from plaso.cli.helpers import virustotal_analysis
 from plaso.cli.helpers import windows_services_analysis

plaso/cli/helpers/database_config.py

@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+"""The arguments helper for a database configuration."""
+
+from plaso.lib import errors
+from plaso.cli.helpers import interface
+from plaso.cli.helpers import server_config
+
+
+class DatabaseConfigHelper(interface.ArgumentsHelper):
+  """CLI arguments helper class for database configuration."""
+
+  NAME = u'database_config'
+  DESCRIPTION = u'Argument helper for a database configuration.'
+
+  _DEFAULT_NAME = u'data'
+  _DEFAULT_PASSWORD = u'toor'
+  _DEFAULT_USERNAME = u'root'
+
+  @classmethod
+  def AddArguments(cls, argument_group):
+    """Add command line arguments the helper supports to an argument group.
+
+    This function takes an argument parser or an argument group object and adds
+    to it all the command line arguments this helper supports.
+
+    Args:
+      argument_group: the argparse group (instance of argparse._ArgumentGroup or
+                      or argparse.ArgumentParser).
+    """
+    argument_group.add_argument(
+        u'--user', dest=u'username', type=unicode, action=u'store',
+        default=None, metavar=u'USERNAME', required=False, help=(
+            u'The username used to connect to the database.'))
+    argument_group.add_argument(
+        u'--password', dest=u'password', type=unicode, action=u'store',
+        default=None, metavar=u'PASSWORD', help=(
+            u'The password for the database user.'))
+    argument_group.add_argument(
+        u'--db_name', '--db-name', dest=u'db_name', action=u'store',
+        type=unicode, default=None, required=False, help=(
+            u'The name of the database to connect to.'))
+
+    server_config.BaseServerConfigHelper.AddArguments(argument_group)
+
+  @classmethod
+  def ParseOptions(cls, options, output_module):
+    """Parses and validates options.
+
+    Args:
+      options: the parser option object (instance of argparse.Namespace).
+      output_module: an output module (instance of OutputModule).
+
+    Raises:
+      BadConfigObject: when the output module object is of the wrong type.
+      BadConfigOption: when a configuration parameter fails validation.
+    """
+    if not hasattr(output_module, u'SetCredentials'):
+      raise errors.BadConfigObject(u'Unable to set username information.')
+
+    username = getattr(options, u'username', None)
+    if not username:
+      username = cls._DEFAULT_USERNAME
+
+    password = getattr(options, u'password', None)
+    if not password:
+      password = cls._DEFAULT_PASSWORD
+
+    name = getattr(options, u'db_name', None)
+    if not name:
+      name = cls._DEFAULT_NAME
+
+    output_module.SetCredentials(
+        username=username, password=password)
+    output_module.SetDatabaseName(name)
+    server_config.BaseServerConfigHelper.ParseOptions(options, output_module)

plaso/cli/helpers/elastic_output.py

@@ -4,19 +4,24 @@
 from plaso.lib import errors
 from plaso.cli.helpers import interface
 from plaso.cli.helpers import manager
+from plaso.cli.helpers import server_config
 from plaso.output import elastic
 
 
+class ElasticServer(server_config.BaseServerConfigHelper):
+  """CLI argument helper for an Elastic Search server."""
+
+  _DEFAULT_SERVER = u'127.0.0.1'
+  _DEFAULT_PORT = 9200
+
+
 class ElasticOutputHelper(interface.ArgumentsHelper):
   """CLI arguments helper class for an Elastic Search output module."""
 
   NAME = u'elastic'
   CATEGORY = u'output'
   DESCRIPTION = u'Argument helper for the Elastic Search output module.'
 
-  DEFAULT_ELASTIC_SERVER = u'127.0.0.1'
-  DEFAULT_ELASTIC_PORT = 9200
-
   @classmethod
   def AddArguments(cls, argument_group):
     """Add command line arguments the helper supports to an argument group.
@@ -38,18 +43,8 @@ def AddArguments(cls, argument_group):
         action=u'store', default=u'', help=(
             u'Name of the document type. This is the name of the document '
             u'type that will be used in ElasticSearch.'))
-    argument_group.add_argument(
-        u'--elastic_server_ip', dest=u'elastic_server', type=unicode,
-        action=u'store', default=u'127.0.0.1', metavar=u'HOSTNAME', help=(
-            u'If the ElasticSearch database resides on a different server '
-            u'than localhost this parameter needs to be passed in. This '
-            u'should be the IP address or the hostname of the server.'))
-    argument_group.add_argument(
-        u'--elastic_port', dest=u'elastic_port', type=int, action=u'store',
-        default=9200, metavar=u'PORT', help=(
-            u'By default ElasticSearch uses the port number 9200, if the '
-            u'database is listening on a different port this parameter '
-            u'can be defined.'))
+
+    ElasticServer.AddArguments(argument_group)
 
   @classmethod
   def ParseOptions(cls, options, output_module):
@@ -69,31 +64,14 @@ def ParseOptions(cls, options, output_module):
 
     output_format = getattr(options, u'output_format', None)
     if output_format != u'elastic':
-      raise errors.WrongHelper(u'Only works on Elastic output module.')
-
-    elastic_server = getattr(options, u'elastic_server', None)
-    if elastic_server is None:
-      raise errors.BadConfigOption(u'Elastic server not set')
-
-    if not elastic_server:
-      elastic_server = cls.DEFAULT_ELASTIC_SERVER
-
-    elastic_port = getattr(options, u'elastic_port', None)
-    if elastic_port is None:
-      raise errors.BadConfigOption(u'Elastic port not set')
-
-    if elastic_port and not isinstance(elastic_port, (int, long)):
-      raise errors.BadConfigOption(u'Elastic port needs to be an integer.')
-
-    if not elastic_port:
-      elastic_port = cls.DEFAULT_ELASTIC_PORT
+      raise errors.BadConfigOption(u'Only works on Elastic output module.')
 
     case_name = getattr(options, u'case_name', None)
     document_type = getattr(options, u'document_type', None)
 
+    ElasticServer.ParseOptions(options, output_module)
     output_module.SetCaseName(case_name)
     output_module.SetDocumentType(document_type)
-    output_module.SetElasticServer(elastic_server, elastic_port)
 
 
 manager.ArgumentHelperManager.RegisterHelper(ElasticOutputHelper)

plaso/cli/helpers/manager.py

@@ -10,7 +10,8 @@ class ArgumentHelperManager(object):
   _helper_classes = {}
 
   @classmethod
-  def AddCommandLineArguments(cls, argument_group, argument_category=None):
+  def AddCommandLineArguments(
+      cls, argument_group, argument_category=None, module_list=None):
     """Adds command line arguments to a configuration object.
 
     Args:
@@ -20,10 +21,18 @@ def AddCommandLineArguments(cls, argument_group, argument_category=None):
                          eg: storage, output. Used to add arguments to a select
                          group of registered helpers. Defaults to None, which
                          applies the added arguments to all helpers.
+      module_list: a list of modules to apply the command line arguments agains.
+                   The comparison is done against the NAME attribute of the
+                   helper. Defaults to None, in which case all registered
+                   helpers are applied.
     """
     for helper in cls._helper_classes.itervalues():
       if argument_category and helper.CATEGORY != argument_category:
         continue
+
+      if module_list and helper.NAME not in module_list:
+        continue
+
       helper.AddArguments(argument_group)
 
   @classmethod

plaso/cli/helpers/manager_test.py

@@ -87,7 +87,7 @@ def testCommandLineArguments(self):
     manager.ArgumentHelperManager.RegisterHelpers(
         [TestHelper, AnotherTestHelper])
 
-    arg_parser = argparse.ArgumentParser()
+    arg_parser = argparse.ArgumentParser(conflict_handler=u'resolve')
     manager.ArgumentHelperManager.AddCommandLineArguments(arg_parser)
 
     # Assert the parameters have been set.

plaso/cli/helpers/mysql_4n6time_output.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+"""The arguments helper for the 4n6time MySQL database output module."""
+
+from plaso.lib import errors
+from plaso.cli.helpers import interface
+from plaso.cli.helpers import database_config
+from plaso.cli.helpers import shared_4n6time_output
+from plaso.cli.helpers import manager
+from plaso.output import mysql_4n6time
+
+
+class MySQL4n6TimeHelper(database_config.DatabaseConfigHelper):
+  """CLI argument helper for a 4n6Time MySQL database server."""
+
+  _DEFAULT_USERNAME = u'root'
+  _DEFAULT_PASSWORD = u'forensic'
+
+
+class MySQL4n6TimeOutputHelper(interface.ArgumentsHelper):
+  """CLI arguments helper class for a MySQL 4n6time output module."""
+
+  NAME = u'4n6time_mysql'
+  CATEGORY = u'output'
+  DESCRIPTION = u'Argument helper for the 4n6Time MySQL output module.'
+
+  @classmethod
+  def AddArguments(cls, argument_group):
+    """Add command line arguments the helper supports to an argument group.
+
+    This function takes an argument parser or an argument group object and adds
+    to it all the command line arguments this helper supports.
+
+    Args:
+      argument_group: the argparse group (instance of argparse._ArgumentGroup or
+                      or argparse.ArgumentParser).
+    """
+    shared_4n6time_output.Shared4n6TimeOutputHelper.AddArguments(argument_group)
+    MySQL4n6TimeHelper.AddArguments(argument_group)
+
+  @classmethod
+  def ParseOptions(cls, options, output_module):
+    """Parses and validates options.
+
+    Args:
+      options: the parser option object (instance of argparse.Namespace).
+      output_module: an output module (instance of OutputModule).
+
+    Raises:
+      BadConfigObject: when the output module object is of the wrong type.
+      BadConfigOption: when a configuration parameter fails validation.
+    """
+    if not isinstance(output_module, mysql_4n6time.MySQL4n6TimeOutputModule):
+      raise errors.BadConfigObject(
+          u'Output module is not an instance of MySQL4n6TimeOutputModule')
+
+    MySQL4n6TimeHelper.ParseOptions(options, output_module)
+    shared_4n6time_output.Shared4n6TimeOutputHelper.ParseOptions(
+        options, output_module)
+
+
+manager.ArgumentHelperManager.RegisterHelper(MySQL4n6TimeOutputHelper)