diff --git a/config/dpkg/changelog b/config/dpkg/changelog index 01481afe06..cd2df0d14d 100644 --- a/config/dpkg/changelog +++ b/config/dpkg/changelog @@ -2,4 +2,4 @@ python-plaso (1.2.1-1) unstable; urgency=low * Auto-generated - -- Log2Timeline Wed, 03 Jun 2015 20:43:58 +0200 + -- Log2Timeline Wed, 03 Jun 2015 12:12:02 -0700 diff --git a/docs/plaso.cli.helpers.rst b/docs/plaso.cli.helpers.rst index 8b6245f5ea..cb165f44e0 100644 --- a/docs/plaso.cli.helpers.rst +++ b/docs/plaso.cli.helpers.rst @@ -4,6 +4,14 @@ plaso.cli.helpers package Submodules ---------- +plaso.cli.helpers.database_config module +---------------------------------------- + +.. automodule:: plaso.cli.helpers.database_config + :members: + :undoc-members: + :show-inheritance: + plaso.cli.helpers.elastic_output module --------------------------------------- @@ -36,6 +44,46 @@ plaso.cli.helpers.manager_test module :undoc-members: :show-inheritance: +plaso.cli.helpers.mysql_4n6time_output module +--------------------------------------------- + +.. automodule:: plaso.cli.helpers.mysql_4n6time_output + :members: + :undoc-members: + :show-inheritance: + +plaso.cli.helpers.pstorage module +--------------------------------- + +.. automodule:: plaso.cli.helpers.pstorage + :members: + :undoc-members: + :show-inheritance: + +plaso.cli.helpers.server_config module +-------------------------------------- + +.. automodule:: plaso.cli.helpers.server_config + :members: + :undoc-members: + :show-inheritance: + +plaso.cli.helpers.shared_4n6time_output module +---------------------------------------------- + +.. automodule:: plaso.cli.helpers.shared_4n6time_output + :members: + :undoc-members: + :show-inheritance: + +plaso.cli.helpers.sqlite_4n6time_output module +---------------------------------------------- + +.. automodule:: plaso.cli.helpers.sqlite_4n6time_output + :members: + :undoc-members: + :show-inheritance: + plaso.cli.helpers.tagging_analysis module ----------------------------------------- @@ -44,6 +92,14 @@ plaso.cli.helpers.tagging_analysis module :undoc-members: :show-inheritance: +plaso.cli.helpers.timesketch_out module +--------------------------------------- + +.. automodule:: plaso.cli.helpers.timesketch_out + :members: + :undoc-members: + :show-inheritance: + plaso.cli.helpers.virustotal_analysis module -------------------------------------------- diff --git a/plaso/cli/helpers/__init__.py b/plaso/cli/helpers/__init__.py index 0ab2f19db7..adb0099516 100644 --- a/plaso/cli/helpers/__init__.py +++ b/plaso/cli/helpers/__init__.py @@ -2,5 +2,9 @@ """This file contains an import statement for each argument helper.""" from plaso.cli.helpers import elastic_output +from plaso.cli.helpers import mysql_4n6time_output +from plaso.cli.helpers import sqlite_4n6time_output +from plaso.cli.helpers import pstorage +from plaso.cli.helpers import timesketch_out from plaso.cli.helpers import virustotal_analysis from plaso.cli.helpers import windows_services_analysis diff --git a/plaso/cli/helpers/database_config.py b/plaso/cli/helpers/database_config.py new file mode 100644 index 0000000000..67cc0a902c --- /dev/null +++ b/plaso/cli/helpers/database_config.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +"""The arguments helper for a database configuration.""" + +from plaso.lib import errors +from plaso.cli.helpers import interface +from plaso.cli.helpers import server_config + + +class DatabaseConfigHelper(interface.ArgumentsHelper): + """CLI arguments helper class for database configuration.""" + + NAME = u'database_config' + DESCRIPTION = u'Argument helper for a database configuration.' + + _DEFAULT_NAME = u'data' + _DEFAULT_PASSWORD = u'toor' + _DEFAULT_USERNAME = u'root' + + @classmethod + def AddArguments(cls, argument_group): + """Add command line arguments the helper supports to an argument group. + + This function takes an argument parser or an argument group object and adds + to it all the command line arguments this helper supports. + + Args: + argument_group: the argparse group (instance of argparse._ArgumentGroup or + or argparse.ArgumentParser). + """ + argument_group.add_argument( + u'--user', dest=u'username', type=unicode, action=u'store', + default=None, metavar=u'USERNAME', required=False, help=( + u'The username used to connect to the database.')) + argument_group.add_argument( + u'--password', dest=u'password', type=unicode, action=u'store', + default=None, metavar=u'PASSWORD', help=( + u'The password for the database user.')) + argument_group.add_argument( + u'--db_name', '--db-name', dest=u'db_name', action=u'store', + type=unicode, default=None, required=False, help=( + u'The name of the database to connect to.')) + + server_config.BaseServerConfigHelper.AddArguments(argument_group) + + @classmethod + def ParseOptions(cls, options, output_module): + """Parses and validates options. + + Args: + options: the parser option object (instance of argparse.Namespace). + output_module: an output module (instance of OutputModule). + + Raises: + BadConfigObject: when the output module object is of the wrong type. + BadConfigOption: when a configuration parameter fails validation. + """ + if not hasattr(output_module, u'SetCredentials'): + raise errors.BadConfigObject(u'Unable to set username information.') + + username = getattr(options, u'username', None) + if not username: + username = cls._DEFAULT_USERNAME + + password = getattr(options, u'password', None) + if not password: + password = cls._DEFAULT_PASSWORD + + name = getattr(options, u'db_name', None) + if not name: + name = cls._DEFAULT_NAME + + output_module.SetCredentials( + username=username, password=password) + output_module.SetDatabaseName(name) + server_config.BaseServerConfigHelper.ParseOptions(options, output_module) diff --git a/plaso/cli/helpers/elastic_output.py b/plaso/cli/helpers/elastic_output.py index 52587e59e1..9a903ecac1 100644 --- a/plaso/cli/helpers/elastic_output.py +++ b/plaso/cli/helpers/elastic_output.py @@ -4,9 +4,17 @@ from plaso.lib import errors from plaso.cli.helpers import interface from plaso.cli.helpers import manager +from plaso.cli.helpers import server_config from plaso.output import elastic +class ElasticServer(server_config.BaseServerConfigHelper): + """CLI argument helper for an Elastic Search server.""" + + _DEFAULT_SERVER = u'127.0.0.1' + _DEFAULT_PORT = 9200 + + class ElasticOutputHelper(interface.ArgumentsHelper): """CLI arguments helper class for an Elastic Search output module.""" @@ -14,9 +22,6 @@ class ElasticOutputHelper(interface.ArgumentsHelper): CATEGORY = u'output' DESCRIPTION = u'Argument helper for the Elastic Search output module.' - DEFAULT_ELASTIC_SERVER = u'127.0.0.1' - DEFAULT_ELASTIC_PORT = 9200 - @classmethod def AddArguments(cls, argument_group): """Add command line arguments the helper supports to an argument group. @@ -38,18 +43,8 @@ def AddArguments(cls, argument_group): action=u'store', default=u'', help=( u'Name of the document type. This is the name of the document ' u'type that will be used in ElasticSearch.')) - argument_group.add_argument( - u'--elastic_server_ip', dest=u'elastic_server', type=unicode, - action=u'store', default=u'127.0.0.1', metavar=u'HOSTNAME', help=( - u'If the ElasticSearch database resides on a different server ' - u'than localhost this parameter needs to be passed in. This ' - u'should be the IP address or the hostname of the server.')) - argument_group.add_argument( - u'--elastic_port', dest=u'elastic_port', type=int, action=u'store', - default=9200, metavar=u'PORT', help=( - u'By default ElasticSearch uses the port number 9200, if the ' - u'database is listening on a different port this parameter ' - u'can be defined.')) + + ElasticServer.AddArguments(argument_group) @classmethod def ParseOptions(cls, options, output_module): @@ -69,31 +64,14 @@ def ParseOptions(cls, options, output_module): output_format = getattr(options, u'output_format', None) if output_format != u'elastic': - raise errors.WrongHelper(u'Only works on Elastic output module.') - - elastic_server = getattr(options, u'elastic_server', None) - if elastic_server is None: - raise errors.BadConfigOption(u'Elastic server not set') - - if not elastic_server: - elastic_server = cls.DEFAULT_ELASTIC_SERVER - - elastic_port = getattr(options, u'elastic_port', None) - if elastic_port is None: - raise errors.BadConfigOption(u'Elastic port not set') - - if elastic_port and not isinstance(elastic_port, (int, long)): - raise errors.BadConfigOption(u'Elastic port needs to be an integer.') - - if not elastic_port: - elastic_port = cls.DEFAULT_ELASTIC_PORT + raise errors.BadConfigOption(u'Only works on Elastic output module.') case_name = getattr(options, u'case_name', None) document_type = getattr(options, u'document_type', None) + ElasticServer.ParseOptions(options, output_module) output_module.SetCaseName(case_name) output_module.SetDocumentType(document_type) - output_module.SetElasticServer(elastic_server, elastic_port) manager.ArgumentHelperManager.RegisterHelper(ElasticOutputHelper) diff --git a/plaso/cli/helpers/manager.py b/plaso/cli/helpers/manager.py index d351081aa0..a46d8ee555 100644 --- a/plaso/cli/helpers/manager.py +++ b/plaso/cli/helpers/manager.py @@ -10,7 +10,8 @@ class ArgumentHelperManager(object): _helper_classes = {} @classmethod - def AddCommandLineArguments(cls, argument_group, argument_category=None): + def AddCommandLineArguments( + cls, argument_group, argument_category=None, module_list=None): """Adds command line arguments to a configuration object. Args: @@ -20,10 +21,18 @@ def AddCommandLineArguments(cls, argument_group, argument_category=None): eg: storage, output. Used to add arguments to a select group of registered helpers. Defaults to None, which applies the added arguments to all helpers. + module_list: a list of modules to apply the command line arguments agains. + The comparison is done against the NAME attribute of the + helper. Defaults to None, in which case all registered + helpers are applied. """ for helper in cls._helper_classes.itervalues(): if argument_category and helper.CATEGORY != argument_category: continue + + if module_list and helper.NAME not in module_list: + continue + helper.AddArguments(argument_group) @classmethod diff --git a/plaso/cli/helpers/manager_test.py b/plaso/cli/helpers/manager_test.py index 7324f5abde..74feb1c392 100644 --- a/plaso/cli/helpers/manager_test.py +++ b/plaso/cli/helpers/manager_test.py @@ -87,7 +87,7 @@ def testCommandLineArguments(self): manager.ArgumentHelperManager.RegisterHelpers( [TestHelper, AnotherTestHelper]) - arg_parser = argparse.ArgumentParser() + arg_parser = argparse.ArgumentParser(conflict_handler=u'resolve') manager.ArgumentHelperManager.AddCommandLineArguments(arg_parser) # Assert the parameters have been set. diff --git a/plaso/cli/helpers/mysql_4n6time_output.py b/plaso/cli/helpers/mysql_4n6time_output.py new file mode 100644 index 0000000000..fe114032e5 --- /dev/null +++ b/plaso/cli/helpers/mysql_4n6time_output.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +"""The arguments helper for the 4n6time MySQL database output module.""" + +from plaso.lib import errors +from plaso.cli.helpers import interface +from plaso.cli.helpers import database_config +from plaso.cli.helpers import shared_4n6time_output +from plaso.cli.helpers import manager +from plaso.output import mysql_4n6time + + +class MySQL4n6TimeHelper(database_config.DatabaseConfigHelper): + """CLI argument helper for a 4n6Time MySQL database server.""" + + _DEFAULT_USERNAME = u'root' + _DEFAULT_PASSWORD = u'forensic' + + +class MySQL4n6TimeOutputHelper(interface.ArgumentsHelper): + """CLI arguments helper class for a MySQL 4n6time output module.""" + + NAME = u'4n6time_mysql' + CATEGORY = u'output' + DESCRIPTION = u'Argument helper for the 4n6Time MySQL output module.' + + @classmethod + def AddArguments(cls, argument_group): + """Add command line arguments the helper supports to an argument group. + + This function takes an argument parser or an argument group object and adds + to it all the command line arguments this helper supports. + + Args: + argument_group: the argparse group (instance of argparse._ArgumentGroup or + or argparse.ArgumentParser). + """ + shared_4n6time_output.Shared4n6TimeOutputHelper.AddArguments(argument_group) + MySQL4n6TimeHelper.AddArguments(argument_group) + + @classmethod + def ParseOptions(cls, options, output_module): + """Parses and validates options. + + Args: + options: the parser option object (instance of argparse.Namespace). + output_module: an output module (instance of OutputModule). + + Raises: + BadConfigObject: when the output module object is of the wrong type. + BadConfigOption: when a configuration parameter fails validation. + """ + if not isinstance(output_module, mysql_4n6time.MySQL4n6TimeOutputModule): + raise errors.BadConfigObject( + u'Output module is not an instance of MySQL4n6TimeOutputModule') + + MySQL4n6TimeHelper.ParseOptions(options, output_module) + shared_4n6time_output.Shared4n6TimeOutputHelper.ParseOptions( + options, output_module) + + +manager.ArgumentHelperManager.RegisterHelper(MySQL4n6TimeOutputHelper) diff --git a/plaso/cli/helpers/pstorage.py b/plaso/cli/helpers/pstorage.py new file mode 100644 index 0000000000..d12a7c3c36 --- /dev/null +++ b/plaso/cli/helpers/pstorage.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +"""The arguments helper for the pstorage output module.""" + +from plaso.lib import errors +from plaso.cli.helpers import interface +from plaso.cli.helpers import manager +from plaso.output import pstorage + + +class PstorageOutputHelper(interface.ArgumentsHelper): + """CLI arguments helper class for a pstorage output module.""" + + NAME = u'pstorage' + CATEGORY = u'output' + DESCRIPTION = u'Argument helper for the pstorage output module.' + + @classmethod + def AddArguments(cls, unused_argument_group): + """Add command line arguments the helper supports to an argument group. + + This function takes an argument parser or an argument group object and adds + to it all the command line arguments this helper supports. + + Args: + argument_group: the argparse group (instance of argparse._ArgumentGroup or + or argparse.ArgumentParser). + """ + pass + + @classmethod + def ParseOptions(cls, options, output_module): + """Parses and validates options. + + Args: + options: the parser option object (instance of argparse.Namespace). + output_module: an output module (instance of OutputModule). + + Raises: + BadConfigObject: when the output module object is of the wrong type. + BadConfigOption: when a configuration parameter fails validation. + """ + if not isinstance(output_module, pstorage.PlasoStorageOutputModule): + raise errors.BadConfigObject( + u'Output module is not an instance of PlasoStorageOutputModule.') + + file_path = getattr(options, u'write', None) + if file_path: + output_module.SetFilehandle(file_path=file_path) + + +manager.ArgumentHelperManager.RegisterHelper(PstorageOutputHelper) diff --git a/plaso/cli/helpers/server_config.py b/plaso/cli/helpers/server_config.py new file mode 100644 index 0000000000..394df447de --- /dev/null +++ b/plaso/cli/helpers/server_config.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +"""The arguments helper for a server configuration.""" + +from plaso.lib import errors +from plaso.cli.helpers import interface + + +class BaseServerConfigHelper(interface.ArgumentsHelper): + """CLI arguments helper class for server configuration.""" + + NAME = u'server_config' + DESCRIPTION = u'Argument helper for a server configuration.' + + _DEFAULT_SERVER = u'127.0.0.1' + _DEFAULT_PORT = 80 + + @classmethod + def AddArguments(cls, argument_group): + """Add command line arguments the helper supports to an argument group. + + This function takes an argument parser or an argument group object and adds + to it all the command line arguments this helper supports. + + Args: + argument_group: the argparse group (instance of argparse._ArgumentGroup or + or argparse.ArgumentParser). + """ + argument_group.add_argument( + u'--server', dest=u'server', type=unicode, action=u'store', + default=None, metavar=u'HOSTNAME', help=( + u'The hostname or server IP address of the server.')) + argument_group.add_argument( + u'--port', dest=u'port', type=int, action=u'store', default=None, + metavar=u'PORT', help=u'The port number of the server.') + + @classmethod + def ParseOptions(cls, options, output_module): + """Parses and validates options. + + Args: + options: the parser option object (instance of argparse.Namespace). + output_module: an output module (instance of OutputModule). + + Raises: + BadConfigObject: when the output module object is of the wrong type. + BadConfigOption: when a configuration parameter fails validation. + """ + if not hasattr(output_module, u'SetServerInformation'): + raise errors.BadConfigObject(u'Unable to set server information.') + + server = getattr(options, u'server', None) + if not server: + server = cls._DEFAULT_SERVER + + port = getattr(options, u'port', None) + if port and not isinstance(port, (int, long)): + raise errors.BadConfigOption(u'Invalid port value not an integer.') + + if not port: + port = cls._DEFAULT_PORT + + output_module.SetServerInformation(server, port) diff --git a/plaso/cli/helpers/shared_4n6time_output.py b/plaso/cli/helpers/shared_4n6time_output.py new file mode 100644 index 0000000000..8b5b5be20d --- /dev/null +++ b/plaso/cli/helpers/shared_4n6time_output.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +"""Arguments helper for information shared between 4n6time output modules.""" + +from plaso.lib import errors +from plaso.cli.helpers import interface +from plaso.output import shared_4n6time + + +class Shared4n6TimeOutputHelper(interface.ArgumentsHelper): + """CLI arguments helper class for 4n6time output modules.""" + + NAME = u'4n6time' + CATEGORY = u'output' + DESCRIPTION = u'Argument helper for shared 4n6Time output modules.' + + _DEFAULT_FIELDS = [ + u'color', u'datetime', u'host', u'source', u'sourcetype', u'user', + u'type'] + + @classmethod + def AddArguments(cls, argument_group): + """Add command line arguments the helper supports to an argument group. + + This function takes an argument parser or an argument group object and adds + to it all the command line arguments this helper supports. + + Args: + argument_group: the argparse group (instance of argparse._ArgumentGroup or + or argparse.ArgumentParser). + """ + argument_group.add_argument( + u'--append', dest=u'append', action=u'store_true', default=False, + required=False, help=( + u'Defines whether the intention is to append to an already ' + u'existing database or overwrite it. Defaults to overwrite.')) + argument_group.add_argument( + u'--evidence', dest=u'evidence', type=unicode, default=u'-', + action=u'store', required=False, help=( + u'Set the evidence field to a specific value, defaults to ' + u'empty.')) + argument_group.add_argument( + u'--fields', dest=u'fields', type=unicode, action=u'store', + nargs=u'*', default=None, help=( + u'Defines which fields should be indexed in the database.')) + + @classmethod + def ParseOptions(cls, options, output_module): + """Parses and validates options. + + Args: + options: the parser option object (instance of argparse.Namespace). + output_module: an output module (instance of OutputModule). + + Raises: + BadConfigObject: when the output module object is of the wrong type. + BadConfigOption: when a configuration parameter fails validation. + """ + if not isinstance(output_module, shared_4n6time.Base4n6TimeOutputModule): + raise errors.BadConfigObject( + u'Output module is not an instance of Base4n6TimeOutputModule') + + append = getattr(options, u'append', False) + evidence = getattr(options, u'evidence', u'-') + + fields = getattr(options, u'fields', None) + if not fields: + fields = cls._DEFAULT_FIELDS + + output_module.SetAppendMode(append) + output_module.SetEvidence(evidence) + output_module.SetFields(fields) diff --git a/plaso/cli/helpers/sqlite_4n6time_output.py b/plaso/cli/helpers/sqlite_4n6time_output.py new file mode 100644 index 0000000000..334713f9b3 --- /dev/null +++ b/plaso/cli/helpers/sqlite_4n6time_output.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +"""The arguments helper for the 4n6time SQLite database output module.""" + +from plaso.lib import errors +from plaso.cli.helpers import interface +from plaso.cli.helpers import shared_4n6time_output +from plaso.cli.helpers import manager +from plaso.output import sqlite_4n6time + + +class SQLite4n6TimeOutputHelper(interface.ArgumentsHelper): + """CLI arguments helper class for a SQLite 4n6time output module.""" + + NAME = u'4n6time_sqlite' + CATEGORY = u'output' + DESCRIPTION = u'Argument helper for the 4n6Time SQLite output module.' + + @classmethod + def AddArguments(cls, argument_group): + """Add command line arguments the helper supports to an argument group. + + This function takes an argument parser or an argument group object and adds + to it all the command line arguments this helper supports. + + Args: + argument_group: the argparse group (instance of argparse._ArgumentGroup or + or argparse.ArgumentParser). + """ + shared_4n6time_output.Shared4n6TimeOutputHelper.AddArguments(argument_group) + + @classmethod + def ParseOptions(cls, options, output_module): + """Parses and validates options. + + Args: + options: the parser option object (instance of argparse.Namespace). + output_module: an output module (instance of OutputModule). + + Raises: + BadConfigObject: when the output module object is of the wrong type. + BadConfigOption: when a configuration parameter fails validation. + """ + if not isinstance(output_module, sqlite_4n6time.SQLite4n6TimeOutputModule): + raise errors.BadConfigObject( + u'Output module is not an instance of SQLite4n6TimeOutputModule') + + shared_4n6time_output.Shared4n6TimeOutputHelper.ParseOptions( + options, output_module) + + filename = getattr(options, u'write', None) + if filename: + output_module.SetFilename(filename) + + +manager.ArgumentHelperManager.RegisterHelper(SQLite4n6TimeOutputHelper) diff --git a/plaso/cli/helpers/timesketch_out.py b/plaso/cli/helpers/timesketch_out.py new file mode 100644 index 0000000000..85f5d8ee74 --- /dev/null +++ b/plaso/cli/helpers/timesketch_out.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +"""The arguments helper for the timesketch output module.""" + +import uuid + +from plaso.lib import errors +from plaso.cli.helpers import interface +from plaso.cli.helpers import manager +from plaso.output import timesketch_out + + +class TimesketchOutputHelper(interface.ArgumentsHelper): + """CLI arguments helper class for a timesketch output module.""" + + NAME = u'timesketch' + CATEGORY = u'output' + DESCRIPTION = u'Argument helper for the timesketch output module.' + + @classmethod + def AddArguments(cls, argument_group): + """Add command line arguments the helper supports to an argument group. + + This function takes an argument parser or an argument group object and adds + to it all the command line arguments this helper supports. + + Args: + argument_group: the argparse group (instance of argparse._ArgumentGroup or + or argparse.ArgumentParser). + """ + default_uid = uuid.uuid4().hex + + argument_group.add_argument( + u'--name', u'--timeline_name', u'--timeline-name', + dest=u'timeline_name', type=unicode, action=u'store', + default=u'', required=False, help=( + u'The name of the timeline in Timesketch. Default: ' + u'hostname if present in the storage file. If no hostname ' + u'is found then manual input is used.')) + argument_group.add_argument( + u'--index', dest=u'index', type=unicode, action=u'store', + default=default_uid, required=False, help=( + u'The name of the Elasticsearch index. Default: Generate a random ' + u'UUID')) + argument_group.add_argument( + u'--flush_interval', '--flush-interval', dest=u'flush_interval', + type=int, action=u'store', default=1000, required=False, help=( + u'The number of events to queue up before sent in bulk ' + u'to Elasticsearch. Default: 1000')) + + @classmethod + def ParseOptions(cls, options, output_module): + """Parses and validates options. + + Args: + options: the parser option object (instance of argparse.Namespace). + output_module: an output module (instance of OutputModule). + + Raises: + BadConfigObject: when the output module object is of the wrong type. + BadConfigOption: when a configuration parameter fails validation. + """ + if not isinstance(output_module, timesketch_out.TimesketchOutputModule): + raise errors.BadConfigObject( + u'Output module is not an instance of TimesketchOutputModule') + + output_format = getattr(options, u'output_format', None) + if output_format != u'timesketch': + raise errors.BadConfigOption(u'Only works on Timesketch output module.') + + flush_interval = getattr(options, u'flush_interval', None) + if flush_interval: + output_module.SetFlushInterval(flush_interval) + + index = getattr(options, u'index', None) + if index: + output_module.SetIndex(index) + + name = getattr(options, u'timeline_name', None) + if name: + output_module.SetName(name) + + +manager.ArgumentHelperManager.RegisterHelper(TimesketchOutputHelper) diff --git a/plaso/filters/dynamic_filter.py b/plaso/filters/dynamic_filter.py index bb7dd868af..b751558f72 100644 --- a/plaso/filters/dynamic_filter.py +++ b/plaso/filters/dynamic_filter.py @@ -142,5 +142,6 @@ def CompileFilter(self, filter_string): if lex.lex_filter: super(DynamicFilter, self).CompileFilter(lex.lex_filter) else: - self.matcher = None + self._matcher = None + self._filter_expression = filter_string diff --git a/plaso/filters/eventfilter.py b/plaso/filters/eventfilter.py index eeec47764c..37d95186c4 100644 --- a/plaso/filters/eventfilter.py +++ b/plaso/filters/eventfilter.py @@ -9,16 +9,17 @@ class EventObjectFilter(filter_interface.FilterObject): def CompileFilter(self, filter_string): """Compile the filter string into a filter matcher.""" - self.matcher = pfilter.GetMatcher(filter_string, True) - if not self.matcher: + self._matcher = pfilter.GetMatcher(filter_string, True) + if not self._matcher: raise errors.WrongPlugin('Malformed filter string.') + self._filter_expression = filter_string def Match(self, event_object): """Evaluate an EventObject against a filter.""" - if not self.matcher: + if not self._matcher: return True - self._decision = self.matcher.Matches(event_object) + self._decision = self._matcher.Matches(event_object) return self._decision diff --git a/plaso/filters/filterlist.py b/plaso/filters/filterlist.py index 08e1811fd6..fc31ef3632 100644 --- a/plaso/filters/filterlist.py +++ b/plaso/filters/filterlist.py @@ -59,6 +59,7 @@ def CompileFilter(self, filter_string): raise errors.WrongPlugin( u'Wrong format of YAML file, entry not a dict ({})'.format( results_type)) + self._filter_expression = filter_string def _ParseEntry(self, entry): """Parse a single YAML filter entry.""" diff --git a/plaso/frontend/plasm.py b/plaso/frontend/plasm.py index 9cc908973a..a605b7b314 100644 --- a/plaso/frontend/plasm.py +++ b/plaso/frontend/plasm.py @@ -540,7 +540,8 @@ def NoDuplicates(self, dump_filename): formatter_mediator, store) output_module = output_manager.OutputManager.NewOutputModule( - u'pstorage', output_mediator_object, filehandle=nodup_filename) + u'pstorage', output_mediator_object) + output_module.SetOutputWriter(nodup_filename) with output_interface.EventBuffer( output_module, check_dedups=True) as output_buffer: diff --git a/plaso/frontend/psort.py b/plaso/frontend/psort.py index 460834c9d6..4fef192c3a 100644 --- a/plaso/frontend/psort.py +++ b/plaso/frontend/psort.py @@ -11,14 +11,11 @@ from plaso.analysis import manager as analysis_manager from plaso.analysis import mediator as analysis_mediator -from plaso.cli import tools as cli_tools -from plaso.cli.helpers import manager as helpers_manager from plaso.engine import knowledge_base from plaso.engine import queue from plaso.frontend import analysis_frontend from plaso.frontend import frontend from plaso.lib import bufferlib -from plaso.lib import errors from plaso.lib import pfilter from plaso.lib import timelib from plaso.multi_processing import multi_process @@ -50,6 +47,8 @@ def __init__(self): self._output_file_object = None self._output_format = None self._preferred_language = u'en-US' + self._quiet_mode = False + self._storage_file = None def _AppendEvent(self, event_object, output_buffer, event_queues): """Appends an event object to an output buffer and queues. @@ -184,21 +183,141 @@ def _ProcessAnalysisPlugins( for item, value in analysis_queue_consumer.counter.iteritems(): counter[item] = value + def SetFilter(self, filter_object, filter_expression): + """Set the filter information. + + Args: + filter_object: a filter object (instance of FilterObject). + filter_expression: the filter expression string. + """ + self._filter_object = filter_object + self._filter_expression = filter_expression + + def SetPreferredLanguageIdentifier(self, language_identifier): + """Sets the preferred language identifier. + + Args: + language_identifier: the language identifier string e.g. en-US for + US English or is-IS for Icelandic. + """ + self._preferred_language = language_identifier + + def SetOutputFilename(self, output_filename): + """Sets the output format. + + Args: + output_filename: the output filename. + """ + self._output_filename = output_filename + + def SetOutputFormat(self, output_format): + """Sets the output format. + + Args: + output_format: the output format. + """ + self._output_format = output_format + + def GetOutputModule( + self, storage_file, preferred_encoding=u'utf-8', timezone=pytz.UTC): + """Return an output module. + + Args: + storage_file: the storage file object (instance of StorageFile). + preferred_encoding: optional preferred encoding. The default is "utf-8". + timezone: optional timezone. The default is UTC. + + Returns: + an output module object (instance of OutputModule) or None if not able to + open one up. + + Raises: + RuntimeError: if a non-recoverable situation is encountered. + """ + formatter_mediator = self.GetFormatterMediator() + + try: + formatter_mediator.SetPreferredLanguageIdentifier( + self._preferred_language) + except (KeyError, TypeError) as exception: + raise RuntimeError(exception) + + output_mediator_object = output_mediator.OutputMediator( + formatter_mediator, storage_file, + preferred_encoding=preferred_encoding, timezone=timezone) + + try: + output_module = output_manager.OutputManager.NewOutputModule( + self._output_format, output_mediator_object) + + except IOError as exception: + raise RuntimeError( + u'Unable to create output module with error: {0:s}'.format( + exception)) + + if not output_module: + raise RuntimeError(u'Missing output module.') + + return output_module + + def GetAnalysisPluginsAndEventQueues(self, analysis_plugins): + """Return a list of analysis plugins and event queues. + + Args: + analysis_plugins: comma separated string with names of analysis + plugins to load. + + Returns: + A tuple of two lists, one containing list of analysis plugins + and the other a list of event queues. + """ + if not analysis_plugins: + return [], [] + + # Start queues and load up plugins. + event_queue_producers = [] + event_queues = [] + analysis_plugins_list = [ + name.strip() for name in analysis_plugins.split(u',')] + + for _ in range(0, len(analysis_plugins_list)): + # TODO: add upper queue limit. + analysis_plugin_queue = multi_process.MultiProcessingQueue(timeout=5) + event_queues.append(analysis_plugin_queue) + event_queue_producers.append( + queue.ItemQueueProducer(event_queues[-1])) + + analysis_plugins = analysis_manager.AnalysisPluginManager.LoadPlugins( + analysis_plugins_list, event_queues) + + return analysis_plugins, event_queue_producers + + def SetQuietMode(self, quiet_mode=False): + """Sets whether tools is in quiet mode or not. + + Args: + quiet_mode: boolean, when True the tool is in quiet mode. + """ + self._quiet_mode = quiet_mode + def ProcessStorage( - self, options, analysis_plugins, deduplicate_events=True, - preferred_encoding=u'utf-8', time_slice=None, - timezone=pytz.UTC, use_time_slicer=False): + self, output_module, storage_file, analysis_plugins, + event_queue_producers, deduplicate_events=True, + preferred_encoding=u'utf-8', time_slice=None, use_time_slicer=False): """Processes a plaso storage file. Args: - options: the command line arguments (instance of argparse.Namespace). - analysis_plugins: the analysis plugins. + output_module: an output module (instance of OutputModule). + storage_file: the storage file object (instance of StorageFile). + analysis_plugins: list of analysis plugin objects (instance of + AnalysisPlugin). + event_queue_producers: list of event queue producer objects (instance + of ItemQueueProducer). deduplicate_events: optional boolean value to indicate if the event objects should be deduplicated. The default is True. preferred_encoding: optional preferred encoding. The default is "utf-8". time_slice: optional time slice object (instance of TimeSlice). The default is None. - timezone: optional timezone. The default is UTC. use_time_slicer: optional boolean value to indicate the 'time slicer' should be used. The default is False. The 'time slicer' will provide a context of events around an event of @@ -211,10 +330,6 @@ def ProcessStorage( Raises: RuntimeError: if a non-recoverable situation is encountered. """ - # TODO: remove this in psort options refactor. - self._output_format = getattr(options, u'output_format', None) - self._output_filename = getattr(options, u'write', None) - if time_slice: if time_slice.event_timestamp: pfilter.TimeRangeCache.SetLowerTimestamp(time_slice.start_timestamp) @@ -223,68 +338,10 @@ def ProcessStorage( elif use_time_slicer: self._filter_buffer = bufferlib.CircularBuffer(time_slice.duration) - if analysis_plugins: - read_only = False - else: - read_only = True - - try: - storage_file = self.OpenStorageFile(read_only=read_only) - except IOError as exception: - raise RuntimeError( - u'Unable to open storage file: {0:s} with error: {1:s}.'.format( - self._storage_file_path, exception)) - counter = None with storage_file: storage_file.SetStoreLimit(self._filter_object) - formatter_mediator = self.GetFormatterMediator() - - try: - formatter_mediator.SetPreferredLanguageIdentifier( - self._preferred_language) - except (KeyError, TypeError) as exception: - raise RuntimeError(exception) - - output_mediator_object = output_mediator.OutputMediator( - formatter_mediator, storage_file, config=options, - preferred_encoding=preferred_encoding, timezone=timezone) - - kwargs = {} - # TODO: refactor this to use CLI argument helpers. - if self._output_format == u'4n6time_sqlite': - kwargs[u'filename'] = self._output_filename - elif self._output_format == u'pstorage': - kwargs[u'filehandle'] = self._output_filename - elif self._output_format not in [u'elastic', u'timesketch']: - if self._output_filename: - self._output_file_object = open(self._output_filename, 'wb') - kwargs[u'output_writer'] = cli_tools.FileObjectOutputWriter( - self._output_file_object) - else: - kwargs[u'output_writer'] = self._output_writer - - try: - output_module = output_manager.OutputManager.NewOutputModule( - self._output_format, output_mediator_object, **kwargs) - - except IOError as exception: - raise RuntimeError( - u'Unable to create output module with error: {0:s}'.format( - exception)) - - if not output_module: - raise RuntimeError(u'Missing output module.') - - # TODO: This should be done in tools/psort.py but requires - # a larger re-factor of this function. - try: - helpers_manager.ArgumentHelperManager.ParseOptions( - options, output_module) - except errors.BadConfigOption as exception: - raise RuntimeError(exception) - # TODO: allow for single processing. # TODO: add upper queue limit. analysis_output_queue = multi_process.MultiProcessingQueue(timeout=5) @@ -328,32 +385,10 @@ def ProcessStorage( # pylint: disable=protected-access storage_file._pre_obj = pre_obj - # Start queues and load up plugins. - event_queue_producers = [] - event_queues = [] - analysis_plugins_list = [ - name.strip() for name in analysis_plugins.split(u',')] - - for _ in xrange(0, len(analysis_plugins_list)): - # TODO: add upper queue limit. - analysis_plugin_queue = multi_process.MultiProcessingQueue(timeout=5) - event_queues.append(analysis_plugin_queue) - event_queue_producers.append( - queue.ItemQueueProducer(event_queues[-1])) - knowledge_base_object = knowledge_base.KnowledgeBase() - analysis_plugins = analysis_manager.AnalysisPluginManager.LoadPlugins( - analysis_plugins_list, event_queues) - # Now we need to start all the plugins. for analysis_plugin in analysis_plugins: - # TODO: This should be done in tools/psort.py but requires - # a larger re-factor of this function. - # Set up the plugin based on the options. - helpers_manager.ArgumentHelperManager.ParseOptions( - options, analysis_plugin) - analysis_report_queue_producer = queue.ItemQueueProducer( analysis_output_queue) @@ -388,8 +423,7 @@ def ProcessStorage( if hasattr(information, u'counter'): counter[u'Stored Events'] += information.counter[u'total'] - # TODO: refactor to separate function. - if not getattr(options, u'quiet', False): + if not self._quiet_mode: logging.info(u'Output processing is done.') # Get all reports and tags from analysis plugins. @@ -543,11 +577,11 @@ class PsortAnalysisProcess(object): """A class to contain information about a running analysis process. Attributes: - completion_event: An optional Event object (instance of - Multiprocessing.Event, Queue.Event or similar) that will - be set when the analysis plugin is complete. - plugin: The plugin running in the process (instance of AnalysisProcess). - process: The process (instance of Multiprocessing.Process) that + completion_event: an optional Event object (instance of + Multiprocessing.Event, Queue.Event or similar) that will + be set when the analysis plugin is complete. + plugin: the plugin running in the process (instance of AnalysisProcess). + process: the process (instance of Multiprocessing.Process) that encapsulates the analysis process. """ def __init__(self, completion_event, plugin, process): diff --git a/plaso/frontend/psort_test.py b/plaso/frontend/psort_test.py index f4662b7e76..de91c5bef4 100644 --- a/plaso/frontend/psort_test.py +++ b/plaso/frontend/psort_test.py @@ -140,22 +140,33 @@ def testReadEntries(self): def testProcessStorage(self): """Test the ProcessStorage function.""" + test_front_end = psort.PsortFrontend() + test_front_end.SetOutputFilename(u'output.txt') + test_front_end.SetOutputFormat(u'dynamic') + test_front_end.SetPreferredLanguageIdentifier(u'en-US') + test_front_end.SetQuietMode(True) + + # TODO: Remove the need to parse the option object at all. options = frontend.Options() + options.data_location = None options.storage_file = self._GetTestFilePath([u'psort_test.out']) - options.output_format = u'dynamic' + test_front_end.ParseOptions(options) - lines = [] - with test_lib.TempDirectory() as temp_directory: - temp_file_name = os.path.join(temp_directory, u'output.txt') - options.write = temp_file_name + storage_file = test_front_end.OpenStorageFile(read_only=True) - test_front_end = psort.PsortFrontend() - test_front_end.ParseOptions(options) - test_front_end.ProcessStorage(options, [], u'text') + output_writer = test_lib.StringIOOutputWriter() + output_module = test_front_end.GetOutputModule(storage_file) + output_module.SetOutputWriter(output_writer) - with open(temp_file_name, 'rb') as file_object: - for line in file_object: - lines.append(line) + counter = test_front_end.ProcessStorage(output_module, storage_file, [], []) + self.assertEqual(counter[u'Stored Events'], 15) + + output_writer.SeekToBeginning() + lines = [] + line = output_writer.GetLine() + while line: + lines.append(line) + line = output_writer.GetLine() self.assertEqual(len(lines), 16) @@ -193,8 +204,8 @@ def testOutput(self): storage_file.store_range = [1] output_mediator_object = output_mediator.OutputMediator( self._formatter_mediator, storage_file) - output_module = TestOutputModule( - output_mediator_object, output_writer=output_writer) + output_module = TestOutputModule(output_mediator_object) + output_module.SetOutputWriter(output_writer) event_buffer = TestEventBuffer( output_module, check_dedups=False, store=storage_file) diff --git a/plaso/lib/filter_interface.py b/plaso/lib/filter_interface.py index db50025f40..e940c2e8e0 100644 --- a/plaso/lib/filter_interface.py +++ b/plaso/lib/filter_interface.py @@ -10,9 +10,17 @@ class FilterObject(object): """The interface that each filter needs to implement in plaso.""" + # TODO: Re-factor into filters/interface and use a manager instead + # of the registry library. __metaclass__ = registry.MetaclassRegistry __abstract = True + @property + def filter_expression(self): + """Return the compiled filter expression or None if not compiled.""" + if self._filter_expression: + return self._filter_expression + @property def filter_name(self): """Return the name of the filter.""" @@ -44,6 +52,12 @@ def limit(self): """Returns the max number of records to return, or zero for all records.""" return 0 + def __init__(self): + """Initialize the filter object.""" + super(FilterObject, self).__init__() + self._filter_expression = None + self._matcher = None + @abc.abstractmethod def CompileFilter(self, unused_filter_string): """Verify filter string and prepare the filter for later usage. diff --git a/plaso/lib/storage.py b/plaso/lib/storage.py index 9563352f73..51280354df 100644 --- a/plaso/lib/storage.py +++ b/plaso/lib/storage.py @@ -263,6 +263,11 @@ def __init__( self._profiling_sample = 0 self._serializers_profiler = None + @property + def file_path(self): + """The file path.""" + return self._output_file + @property def serialization_format(self): """The serialization format.""" diff --git a/plaso/output/dynamic.py b/plaso/output/dynamic.py index cf3d323e55..a0c22968e3 100644 --- a/plaso/output/dynamic.py +++ b/plaso/output/dynamic.py @@ -54,32 +54,15 @@ class DynamicOutputModule(interface.LinearOutputModule): u'zone': u'_FormatZone', } - def __init__( - self, output_mediator, fields_filter=None, output_writer=None, **kwargs): + def __init__(self, output_mediator): """Initializes the output module object. Args: output_mediator: The output mediator object (instance of OutputMediator). - fields_filter: optional filter object (instance of FilterObject) to - indicate which fields should be outputed. The default - is None. - output_writer: Optional output writer object (instance of - CLIOutputWriter). The default is None. """ - super(DynamicOutputModule, self).__init__( - output_mediator, output_writer=output_writer, **kwargs) - self._fields = None - self._field_delimiter = None - - if fields_filter: - self._fields = fields_filter.fields - self._field_delimiter = fields_filter.separator - - if not self._fields: - self._fields = self._DEFAULT_FIELDS - - if not self._field_delimiter: - self._field_delimiter = self._FIELD_DELIMITER + super(DynamicOutputModule, self).__init__(output_mediator) + self._field_delimiter = self._FIELD_DELIMITER + self._fields = self._DEFAULT_FIELDS def _FormatDate(self, event_object): """Formats the date. @@ -342,6 +325,19 @@ def _SanitizeField(self, field): return field.replace(self._field_delimiter, u' ') return field + def SetFieldsFilter(self, fields_filter): + """Set the fields filter. + + Args: + fields_filter: filter object (instance of FilterObject) to + indicate which fields should be outputed. + """ + if not fields_filter: + return + + self._fields = fields_filter.fields + self._field_delimiter = fields_filter.separator + def WriteEventBody(self, event_object): """Writes the body of an event object to the output. diff --git a/plaso/output/dynamic_test.py b/plaso/output/dynamic_test.py index 7d950b398b..7ea2241923 100644 --- a/plaso/output/dynamic_test.py +++ b/plaso/output/dynamic_test.py @@ -52,8 +52,8 @@ def testHeader(self): """Tests the WriteHeader function.""" output_mediator = self._CreateOutputMediator() output_writer = cli_test_lib.TestOutputWriter() - output_module = dynamic.DynamicOutputModule( - output_mediator, output_writer=output_writer) + output_module = dynamic.DynamicOutputModule(output_mediator) + output_module.SetOutputWriter(output_writer) expected_header = ( b'datetime,timestamp_desc,source,source_long,message,parser,' b'display_name,tag,store_number,store_index\n') @@ -66,9 +66,9 @@ def testHeader(self): u'date', u'time', u'message', u'hostname', u'filename', u'some_stuff']) output_mediator = self._CreateOutputMediator() output_writer = cli_test_lib.TestOutputWriter() - output_module = dynamic.DynamicOutputModule( - output_mediator, fields_filter=filter_object, - output_writer=output_writer) + output_module = dynamic.DynamicOutputModule(output_mediator) + output_module.SetFieldsFilter(filter_object) + output_module.SetOutputWriter(output_writer) expected_header = b'date,time,message,hostname,filename,some_stuff\n' output_module.WriteHeader() @@ -80,9 +80,9 @@ def testHeader(self): separator='@') output_mediator = self._CreateOutputMediator() output_writer = cli_test_lib.TestOutputWriter() - output_module = dynamic.DynamicOutputModule( - output_mediator, fields_filter=filter_object, - output_writer=output_writer) + output_module = dynamic.DynamicOutputModule(output_mediator) + output_module.SetFieldsFilter(filter_object) + output_module.SetOutputWriter(output_writer) expected_header = b'date@time@message@hostname@filename@some_stuff\n' output_module.WriteHeader() @@ -102,9 +102,9 @@ def testWriteEventBody(self): u'filename', u'inode', u'notes', u'format', u'extra']) output_mediator = self._CreateOutputMediator() output_writer = cli_test_lib.TestOutputWriter() - output_module = dynamic.DynamicOutputModule( - output_mediator, fields_filter=filter_object, - output_writer=output_writer) + output_module = dynamic.DynamicOutputModule(output_mediator) + output_module.SetFieldsFilter(filter_object) + output_module.SetOutputWriter(output_writer) output_module.WriteHeader() expected_header = ( @@ -127,9 +127,9 @@ def testWriteEventBody(self): u'datetime', u'nonsense', u'hostname', u'message']) output_mediator = self._CreateOutputMediator() output_writer = cli_test_lib.TestOutputWriter() - output_module = dynamic.DynamicOutputModule( - output_mediator, fields_filter=filter_object, - output_writer=output_writer) + output_module = dynamic.DynamicOutputModule(output_mediator) + output_module.SetFieldsFilter(filter_object) + output_module.SetOutputWriter(output_writer) expected_header = b'datetime,nonsense,hostname,message\n' output_module.WriteHeader() diff --git a/plaso/output/elastic.py b/plaso/output/elastic.py index 0ad480169d..058a3d3a80 100644 --- a/plaso/output/elastic.py +++ b/plaso/output/elastic.py @@ -23,13 +23,13 @@ class ElasticSearchOutputModule(interface.OutputModule): NAME = u'elastic' DESCRIPTION = u'Saves the events into an ElasticSearch database.' - def __init__(self, output_mediator, **kwargs): + def __init__(self, output_mediator): """Initializes the output module object. Args: output_mediator: The output mediator object (instance of OutputMediator). """ - super(ElasticSearchOutputModule, self).__init__(output_mediator, **kwargs) + super(ElasticSearchOutputModule, self).__init__(output_mediator) self._counter = 0 self._data = [] self._doc_type = None @@ -126,7 +126,7 @@ def SetDocumentType(self, document_type): else: self._doc_type = u'event' - def SetElasticServer(self, elastic_host, elastic_port): + def SetServerInformation(self, elastic_host, elastic_port): """Set the ElasticSearch connection. Args: diff --git a/plaso/output/interface.py b/plaso/output/interface.py index 40ee77204d..1ca0eac885 100644 --- a/plaso/output/interface.py +++ b/plaso/output/interface.py @@ -11,38 +11,18 @@ class OutputModule(object): """Class that implements the output module object interface.""" - # TODO: refactor this to the cli classes (aka storage helper). - # Optional arguments to be added to the argument parser. - # An example would be: - # ARGUMENTS = [('--myparameter', { - # 'action': 'store', - # 'help': 'This is my parameter help', - # 'dest': 'myparameter', - # 'default': '', - # 'type': 'unicode'})] - # - # Where all arguments into the dict object have a direct translation - # into the argparse parser. - ARGUMENTS = [] - NAME = u'' DESCRIPTION = u'' - def __init__(self, output_mediator, **kwargs): + def __init__(self, output_mediator): """Initializes the output module object. Args: output_mediator: The output mediator object (instance of OutputMediator). - kwargs: a dictionary of keyword arguments dependending on the output - module. Raises: ValueError: when there are unused keyword arguments. """ - if kwargs: - raise ValueError(u'Unused keyword arguments: {0:s}.'.format( - u', '.join(kwargs.keys()))) - super(OutputModule, self).__init__() self._output_mediator = output_mediator @@ -50,6 +30,15 @@ def Close(self): """Closes the output.""" pass + def GetMissingArguments(self): + """Return a list of arguments that are missing from the input. + + Returns: + A list of argument names that are missing and necessary for the + module to continue to operate. + """ + return [] + def Open(self): """Opens the output.""" pass @@ -120,22 +109,17 @@ class LinearOutputModule(OutputModule): # classes need to implement that function. # pylint: disable=abstract-method - def __init__(self, output_mediator, output_writer=None, **kwargs): + def __init__(self, output_mediator): """Initializes the output module object. Args: output_mediator: The output mediator object (instance of OutputMediator). - output_writer: Optional output writer object (instance of - CLIOutputWriter). The default is None. Raises: ValueError: if the output writer is missing. """ - if not output_writer: - raise ValueError(u'Missing output writer.') - - super(LinearOutputModule, self).__init__(output_mediator, **kwargs) - self._output_writer = output_writer + super(LinearOutputModule, self).__init__(output_mediator) + self._output_writer = None def _WriteLine(self, line): """Write a single line to the supplied file-like object. @@ -145,6 +129,15 @@ def _WriteLine(self, line): """ self._output_writer.Write(line) + def SetOutputWriter(self, output_writer): + """Set the output writer. + + Args: + output_writer: Optional output writer object (instance of + CLIOutputWriter). The default is None. + """ + self._output_writer = output_writer + def Close(self): """Closes the output.""" self._output_writer = None diff --git a/plaso/output/interface_test.py b/plaso/output/interface_test.py index aeacfd7ac7..f43ecece47 100644 --- a/plaso/output/interface_test.py +++ b/plaso/output/interface_test.py @@ -70,8 +70,8 @@ def testOutput(self): output_mediator = self._CreateOutputMediator() output_writer = cli_test_lib.TestOutputWriter() - output_module = TestOutputModule( - output_mediator, output_writer=output_writer) + output_module = TestOutputModule(output_mediator) + output_module.SetOutputWriter(output_writer) output_module.WriteHeader() for event_object in events: output_module.WriteEvent(event_object) @@ -140,8 +140,8 @@ def testFlush(self): """Test to ensure we empty our buffers and sends to output properly.""" output_mediator = self._CreateOutputMediator() output_writer = cli_test_lib.TestOutputWriter() - output_module = TestOutputModule( - output_mediator, output_writer=output_writer) + output_module = TestOutputModule(output_mediator) + output_module.SetOutputWriter(output_writer) event_buffer = interface.EventBuffer(output_module, False) event_buffer.Append(TestEvent(123456, u'Now is now')) diff --git a/plaso/output/json_out.py b/plaso/output/json_out.py index 117455664f..6c90dcd1e7 100644 --- a/plaso/output/json_out.py +++ b/plaso/output/json_out.py @@ -12,13 +12,13 @@ class JSONOutputModule(interface.LinearOutputModule): NAME = u'json' DESCRIPTION = u'Saves the events into a JSON format.' - def __init__(self, output_mediator, **kwargs): + def __init__(self, output_mediator): """Initializes the output module object. Args: output_mediator: The output mediator object (instance of OutputMediator). """ - super(JSONOutputModule, self).__init__(output_mediator, **kwargs) + super(JSONOutputModule, self).__init__(output_mediator) self._event_counter = 0 def WriteEventBody(self, event_object): diff --git a/plaso/output/json_out_test.py b/plaso/output/json_out_test.py index fe6fcb6e17..ee0e7bcf27 100644 --- a/plaso/output/json_out_test.py +++ b/plaso/output/json_out_test.py @@ -20,8 +20,8 @@ def setUp(self): """Sets up the objects needed for this test.""" output_mediator = self._CreateOutputMediator() self._output_writer = cli_test_lib.TestOutputWriter() - self._output_module = json_out.JSONOutputModule( - output_mediator, output_writer=self._output_writer) + self._output_module = json_out.JSONOutputModule(output_mediator) + self._output_module.SetOutputWriter(self._output_writer) self._event_object = test_lib.TestEventObject() def testWriteHeader(self): diff --git a/plaso/output/l2t_csv_test.py b/plaso/output/l2t_csv_test.py index 6f43812b11..6ccf834fe6 100644 --- a/plaso/output/l2t_csv_test.py +++ b/plaso/output/l2t_csv_test.py @@ -49,8 +49,8 @@ def setUp(self): """Sets up the objects needed for this test.""" output_mediator = self._CreateOutputMediator() self._output_writer = cli_test_lib.TestOutputWriter() - self.formatter = l2t_csv.L2TCSVOutputModule( - output_mediator, output_writer=self._output_writer) + self.formatter = l2t_csv.L2TCSVOutputModule(output_mediator) + self.formatter.SetOutputWriter(self._output_writer) self.event_object = L2tTestEvent() def testWriteHeader(self): diff --git a/plaso/output/manager.py b/plaso/output/manager.py index b6cd394057..00568b3ec9 100644 --- a/plaso/output/manager.py +++ b/plaso/output/manager.py @@ -103,7 +103,7 @@ def HasOutputClass(cls, name): return name.lower() in cls._output_classes @classmethod - def NewOutputModule(cls, name, output_mediator, **kwargs): + def NewOutputModule(cls, name, output_mediator): """Creates a new output module object for the specified output format. Args: @@ -118,7 +118,7 @@ def NewOutputModule(cls, name, output_mediator, **kwargs): ValueError: if name is not a string. """ output_class = cls.GetOutputClass(name) - return output_class(output_mediator, **kwargs) + return output_class(output_mediator) @classmethod def RegisterOutput(cls, output_class, disabled=False): diff --git a/plaso/output/mediator.py b/plaso/output/mediator.py index b98485820d..6856c7bbdf 100644 --- a/plaso/output/mediator.py +++ b/plaso/output/mediator.py @@ -11,8 +11,8 @@ class OutputMediator(object): """Class that implements the output mediator.""" def __init__( - self, formatter_mediator, storage_object, config=None, - fields_filter=None, preferred_encoding=u'utf-8', timezone=pytz.UTC): + self, formatter_mediator, storage_object, fields_filter=None, + preferred_encoding=u'utf-8', timezone=pytz.UTC): """Initializes a output mediator object. Args: @@ -20,8 +20,6 @@ def __init__( FormatterMediator). storage_object: a storage file object (instance of StorageFile) that defines the storage. - config: optional configuration object, containing config information. - The default is None. fields_filter: optional filter object (instance of FilterObject) to indicate which fields should be outputed. The default is None. @@ -29,7 +27,6 @@ def __init__( timezone: optional timezone. The default is UTC. """ super(OutputMediator, self).__init__() - self._config = config self._formatter_mediator = formatter_mediator self._hostnames = None self._preferred_encoding = preferred_encoding @@ -74,23 +71,22 @@ def encoding(self): return self._preferred_encoding @property - def timezone(self): - """The timezone.""" - return self._timezone + def filter_expression(self): + """The filter expression if a filter is set, None otherwise.""" + if not self.fields_filter: + return - # TODO: solve this differently in a future refactor. - def GetConfigurationValue(self, identifier, default_value=None): - """Retrieves a configuration value. + return self.fields_filter.filter_expression - Args: - identifier: the identifier of the configuration value. - default_value: optional value containing the default. - The default is None. + @property + def storage_file_path(self): + """The storage file path.""" + return self._storage_object.file_path - Returns: - The configuration value or None if not set. - """ - return getattr(self._config, identifier, default_value) + @property + def timezone(self): + """The timezone.""" + return self._timezone def GetEventFormatter(self, event_object): """Retrieves the event formatter for a specific event object type. diff --git a/plaso/output/mediator_test.py b/plaso/output/mediator_test.py index 3f03185749..078bafb464 100644 --- a/plaso/output/mediator_test.py +++ b/plaso/output/mediator_test.py @@ -5,7 +5,6 @@ import unittest from plaso.output import mediator -from plaso.output import test_lib class OutputMediatorTest(unittest.TestCase): @@ -16,22 +15,6 @@ def testInitialization(self): output_mediator = mediator.OutputMediator(None, None) self.assertNotEqual(output_mediator, None) - def testGetConfigurationValue(self): - """Tests the GetConfigurationValue function.""" - expected_config_value = u'My test config setting.' - - config = test_lib.TestConfig() - config.my_setting = expected_config_value - - output_mediator = mediator.OutputMediator(None, None, config=config) - self.assertNotEqual(output_mediator, None) - - config_value = output_mediator.GetConfigurationValue(u'my_setting') - self.assertEqual(config_value, expected_config_value) - - config_value = output_mediator.GetConfigurationValue(u'bogus') - self.assertEqual(config_value, None) - # TODO: add more tests: # GetEventFormatter test. # GetFormattedMessages test. diff --git a/plaso/output/mysql_4n6time.py b/plaso/output/mysql_4n6time.py index 0538e7e9eb..944dfe15a0 100644 --- a/plaso/output/mysql_4n6time.py +++ b/plaso/output/mysql_4n6time.py @@ -18,66 +18,6 @@ class MySQL4n6TimeOutputModule(shared_4n6time.Base4n6TimeOutputModule): NAME = '4n6time_mysql' DESCRIPTION = u'MySQL database output for the 4n6time tool.' - # TODO: move this to a CLI argument helper. - ARGUMENTS = [ - ('--db_user', { - 'dest': 'db_user', - 'type': unicode, - 'help': 'Defines the database user.', - 'metavar': 'USERNAME', - 'action': 'store', - 'default': 'root'}), - ('--db_host', { - 'dest': 'db_host', - 'metavar': 'HOSTNAME', - 'type': unicode, - 'help': ( - 'Defines the IP address or the hostname of the database ' - 'server.'), - 'action': 'store', - 'default': 'localhost'}), - ('--db_pass', { - 'dest': 'db_pass', - 'metavar': 'PASSWORD', - 'type': unicode, - 'help': 'The password for the database user.', - 'action': 'store', - 'default': 'forensic'}), - ('--db_name', { - 'dest': 'db_name', - 'type': unicode, - 'help': 'The name of the database to connect to.', - 'action': 'store', - 'default': 'log2timeline'}), - ('--append', { - 'dest': 'append', - 'action': 'store_true', - 'help': ( - 'Defines whether the intention is to append to an already ' - 'existing database or overwrite it. Defaults to overwrite.'), - 'default': False}), - ('--fields', { - 'dest': 'fields', - 'action': 'store', - 'type': unicode, - 'nargs': '*', - 'help': 'Defines which fields should be indexed in the database.', - 'default': [ - 'host', 'user', 'source', 'sourcetype', 'type', 'datetime', - 'color']}), - ('--evidence', { - 'dest': 'evidence', - 'action': 'store', - 'help': ( - 'Set the evidence field to a specific value, defaults to ' - 'empty.'), - 'type': unicode, - 'default': '-'})] - - _DEFAULT_FIELDS = frozenset([ - u'host', u'user', u'source', u'sourcetype', u'type', u'datetime', - u'color']) - _META_FIELDS = frozenset([ u'sourcetype', u'source', u'user', u'host', u'MACB', u'color', u'type', u'record_number']) @@ -114,33 +54,21 @@ class MySQL4n6TimeOutputModule(shared_4n6time.Base4n6TimeOutputModule): u':URL, :record_number, :event_identifier, :event_type,' u':source_name, :user_sid, :computer_name, :evidence)') - def __init__(self, output_mediator, **kwargs): + def __init__(self, output_mediator): """Initializes the output module object. Args: output_mediator: The output mediator object (instance of OutputMediator). """ - super(MySQL4n6TimeOutputModule, self).__init__(output_mediator, **kwargs) + super(MySQL4n6TimeOutputModule, self).__init__(output_mediator) self._connection = None self._cursor = None - self._append = self._output_mediator.GetConfigurationValue( - u'append', default_value=False) - self._dbname = self._output_mediator.GetConfigurationValue( - u'db_name', default_value=u'log2timeline') - self._evidence = self._output_mediator.GetConfigurationValue( - u'evidence', default_value=u'-') - self._fields = self._output_mediator.GetConfigurationValue( - u'fields', default_value=self._DEFAULT_FIELDS) - self._host = self._output_mediator.GetConfigurationValue( - u'db_host', default_value=u'localhost') - self._password = self._output_mediator.GetConfigurationValue( - u'db_pass', default_value=u'forensic') - self._set_status = self._output_mediator.GetConfigurationValue( - u'set_status') - self._user = self._output_mediator.GetConfigurationValue( - u'db_user', default_value=u'root') + self._dbname = u'log2timeline' + self._host = u'localhost' + self._password = u'forensic' + self._user = u'root' def _GetDistinctValues(self, field_name): """Query database for unique field types. @@ -296,6 +224,36 @@ def Open(self): self.count = 0 + def SetCredentials(self, username=None, password=None): + """Set the database credentials. + + Args: + username: an optional username field. Defaults to None. + password: an optional password field. Defaults to None. + """ + if username: + self._user = username + if password: + self._password = password + + def SetDatabaseName(self, name): + """Set the database name. + + Args: + name: the database name. + """ + self._dbname = name + + def SetServerInformation(self, server, port): + """Set the server information. + + Args: + server: the server name or IP address. + port: the port number the database listens on. + """ + self._host = server + self._port = port + def WriteEventBody(self, event_object): """Writes the body of an event object to the output. diff --git a/plaso/output/pstorage.py b/plaso/output/pstorage.py index fac0b7a041..04cb176fb6 100644 --- a/plaso/output/pstorage.py +++ b/plaso/output/pstorage.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- """Implements a StorageFile output module.""" -import sys - from plaso.lib import event from plaso.lib import storage from plaso.lib import timelib @@ -16,29 +14,15 @@ class PlasoStorageOutputModule(interface.OutputModule): NAME = u'pstorage' DESCRIPTION = u'Dumps event objects to a plaso storage file.' - def __init__(self, output_mediator, filehandle=sys.stdout, **kwargs): + def __init__(self, output_mediator): """Initializes the output module object. Args: output_mediator: The output mediator object (instance of OutputMediator). - filehandle: Optional file-like object that can be written to. - The default is sys.stdout. - - Raises: - TypeError: if the file handle is of an unsupported type. """ - super(PlasoStorageOutputModule, self).__init__(output_mediator, **kwargs) - - if isinstance(filehandle, basestring): - self._file_object = open(filehandle, 'wb') - - # Check if the filehandle object has a write method. - elif hasattr(filehandle, u'write'): - self._file_object = filehandle - - else: - raise TypeError(u'Unsupported file handle type.') + super(PlasoStorageOutputModule, self).__init__(output_mediator) + self._file_object = None self._storage = None def Close(self): @@ -51,17 +35,30 @@ def Open(self): pre_obj.collection_information = { u'time_of_run': timelib.Timestamp.GetNow()} - configuration_value = self._output_mediator.GetConfigurationValue(u'filter') - if configuration_value: - pre_obj.collection_information[u'filter'] = configuration_value + filter_expression = self._output_mediator.filter_expression + if filter_expression: + pre_obj.collection_information[u'filter'] = filter_expression - configuration_value = self._output_mediator.GetConfigurationValue( - u'storagefile') - if configuration_value: - pre_obj.collection_information[u'file_processed'] = configuration_value + storage_file_path = self._output_mediator.storage_file_path + if storage_file_path: + pre_obj.collection_information[u'file_processed'] = storage_file_path self._storage = storage.StorageFile(self._file_object, pre_obj=pre_obj) + def SetFilehandle(self, file_path=None, file_object=None): + """Sets the filehandle. + + Args: + file_path: the full path to the output file. + file_object: a file like object to use for a filehandle. + """ + if file_object: + self._file_object = file_object + return + + if file_path: + self._file_object = open(file_path, 'wb') + def WriteEventBody(self, event_object): """Writes the body of an event object to the output. diff --git a/plaso/output/pstorage_test.py b/plaso/output/pstorage_test.py index 8e4453ab9b..64af1b8a9f 100644 --- a/plaso/output/pstorage_test.py +++ b/plaso/output/pstorage_test.py @@ -49,8 +49,8 @@ def testOutput(self): # Copy events to pstorage dump. with storage.StorageFile(self.test_filename, read_only=True) as store: output_mediator = self._CreateOutputMediator(storage_object=store) - formatter = pstorage.PlasoStorageOutputModule( - output_mediator, filehandle=dump_file) + formatter = pstorage.PlasoStorageOutputModule(output_mediator) + formatter.SetFilehandle(dump_file) with interface.EventBuffer( formatter, check_dedups=False) as output_buffer: diff --git a/plaso/output/rawpy_test.py b/plaso/output/rawpy_test.py index 89838b1bda..1665d54d32 100644 --- a/plaso/output/rawpy_test.py +++ b/plaso/output/rawpy_test.py @@ -17,8 +17,8 @@ def setUp(self): """Sets up the objects needed for this test.""" output_mediator = self._CreateOutputMediator() self._output_writer = cli_test_lib.TestOutputWriter() - self._output_module = rawpy.NativePythonOutputModule( - output_mediator, output_writer=self._output_writer) + self._output_module = rawpy.NativePythonOutputModule(output_mediator) + self._output_module.SetOutputWriter(self._output_writer) self._event_object = test_lib.TestEventObject() def testWriteEventBody(self): diff --git a/plaso/output/shared_4n6time.py b/plaso/output/shared_4n6time.py index f0d8e4220f..b5d8c4c4c0 100644 --- a/plaso/output/shared_4n6time.py +++ b/plaso/output/shared_4n6time.py @@ -16,19 +16,24 @@ class Base4n6TimeOutputModule(interface.OutputModule): NAME = '4n6time_shared' - def __init__(self, output_mediator, **kwargs): + _DEFAULT_FIELDS = [ + u'color', u'datetime', u'host', u'source', u'sourcetype', u'user', + u'type'] + + def __init__(self, output_mediator): """Initializes the output module object. Args: output_mediator: The output mediator object (instance of OutputMediator). - kwargs: a dictionary of keyword arguments dependending on the output - module. Raises: ValueError: when there are unused keyword arguments. """ - super(Base4n6TimeOutputModule, self).__init__(output_mediator, **kwargs) - self._evidence = None + super(Base4n6TimeOutputModule, self).__init__(output_mediator) + self._append = False + self._evidence = u'-' + self._fields = self._DEFAULT_FIELDS + self._set_status = None def _GetSanitizedEventValues(self, event_object): """Sanitizes the event object for use in 4n6time. @@ -148,3 +153,38 @@ def _GetVSSNumber(self, event_object): return -1 return getattr(event_object.pathspec, u'vss_store_number', -1) + + def SetAppendMode(self, append): + """Set the append status. + + Args: + append: boolean that determines whether or not to append to the database. + """ + if append: + self._append = True + else: + self._append = False + + def SetEvidence(self, evidence): + """Set the evidence field. + + Args: + evidence: the evidence field. + """ + self._evidence = evidence + + def SetFields(self, fields): + """Set the fields that will be indexed in the database. + + Args: + fields: a list of fields that should be indexed. + """ + self._fields = fields + + def SetStatusObject(self, status_object): + """Set the status object. + + Args: + status_object: status object provided by the 4n6time tool. + """ + self._set_status = status_object diff --git a/plaso/output/sqlite_4n6time.py b/plaso/output/sqlite_4n6time.py index 84021ca44f..95bd546457 100644 --- a/plaso/output/sqlite_4n6time.py +++ b/plaso/output/sqlite_4n6time.py @@ -16,10 +16,6 @@ class SQLite4n6TimeOutputModule(shared_4n6time.Base4n6TimeOutputModule): DESCRIPTION = ( u'Saves the data in a SQLite database, used by the tool 4n6time.') - _DEFAULT_FIELDS = frozenset([ - u'host', u'user', u'source', u'sourcetype', u'type', u'datetime', - u'color']) - _META_FIELDS = frozenset([ u'sourcetype', u'source', u'user', u'host', u'MACB', u'color', u'type', u'record_number']) @@ -51,34 +47,19 @@ class SQLite4n6TimeOutputModule(shared_4n6time.Base4n6TimeOutputModule): u':URL, :record_number, :event_identifier, :event_type,' u':source_name, :user_sid, :computer_name, :evidence)') - def __init__(self, output_mediator, filename=None, **kwargs): + def __init__(self, output_mediator): """Initializes the output module object. Args: output_mediator: The output mediator object (instance of OutputMediator). - filename: The filename. Raises: ValueError: if the file handle is missing. """ - if not filename: - raise ValueError(u'Missing filename.') - - super(SQLite4n6TimeOutputModule, self).__init__(output_mediator, **kwargs) - self._append = self._output_mediator.GetConfigurationValue( - u'append', default_value=False) - + super(SQLite4n6TimeOutputModule, self).__init__(output_mediator) self._connection = None self._cursor = None - - self._evidence = self._output_mediator.GetConfigurationValue( - u'evidence', default_value=u'-') - self._fields = self._output_mediator.GetConfigurationValue( - u'fields', default_value=self._DEFAULT_FIELDS) - - self._filename = filename - self._set_status = self._output_mediator.GetConfigurationValue( - u'set_status') + self._filename = None def _GetDistinctValues(self, field_name): """Query database for unique field types. @@ -205,6 +186,14 @@ def Open(self): self.count = 0 + def SetFilename(self, filename): + """Sets the filename. + + Args: + filename: the filename. + """ + self._filename = filename + def WriteEventBody(self, event_object): """Writes the body of an event object to the output. diff --git a/plaso/output/sqlite_4n6time_test.py b/plaso/output/sqlite_4n6time_test.py index dd2becb3d0..83564acee3 100644 --- a/plaso/output/sqlite_4n6time_test.py +++ b/plaso/output/sqlite_4n6time_test.py @@ -92,7 +92,8 @@ def dict_from_row(row): temp_file = os.path.join(dirname, u'sqlite_4n6.out') output_mediator = self._CreateOutputMediator(storage_object=temp_file) self._sqlite_output = sqlite_4n6time.SQLite4n6TimeOutputModule( - output_mediator, filename=temp_file) + output_mediator) + self._sqlite_output.SetFilename(temp_file) self._sqlite_output.Open() self._sqlite_output.WriteEventBody(self._event_object) diff --git a/plaso/output/test_lib.py b/plaso/output/test_lib.py index 17a7962cae..6932951168 100644 --- a/plaso/output/test_lib.py +++ b/plaso/output/test_lib.py @@ -79,12 +79,10 @@ class OutputModuleTestCase(unittest.TestCase): # conventions. maxDiff = None - def _CreateOutputMediator(self, config=None, storage_object=None): + def _CreateOutputMediator(self, storage_object=None): """Creates a test output mediator. Args: - config: optional configuration object, containing config information. - The default is None. storage_object: optional storage file object (instance of StorageFile) that defines the storage. The default is None. @@ -92,5 +90,4 @@ def _CreateOutputMediator(self, config=None, storage_object=None): An output mediator (instance of OutputMediator). """ formatter_mediator = formatters_mediator.FormatterMediator() - return mediator.OutputMediator( - formatter_mediator, storage_object, config=config) + return mediator.OutputMediator(formatter_mediator, storage_object) diff --git a/plaso/output/timesketch_out.py b/plaso/output/timesketch_out.py index 5c21b404d0..3423614c07 100644 --- a/plaso/output/timesketch_out.py +++ b/plaso/output/timesketch_out.py @@ -3,7 +3,6 @@ from collections import Counter import logging -import uuid try: from elasticsearch import exceptions as elastic_exceptions @@ -31,71 +30,34 @@ class TimesketchOutputModule(interface.OutputModule): NAME = u'timesketch' DESCRIPTION = u'Create a Timesketch timeline.' - ARGUMENTS = [ - (u'--name', { - u'dest': u'name', - u'type': unicode, - u'help': (u'The name of the timeline in Timesketch. Default: ' - u'hostname if present in the storage file. If no hostname ' - u'is found then manual input is used.'), - u'action': u'store', - u'required': False, - u'default': u''}), - (u'--index', { - u'dest': u'index', - u'type': unicode, - u'help': u'The name of the Elasticsearch index. Default: Generate a ' - u'random UUID', - u'action': u'store', - u'required': False, - u'default': uuid.uuid4().hex}), - (u'--flush_interval', { - u'dest': u'flush_interval', - u'type': int, - u'help': u'The number of events to queue up before sent in bulk ' - u'to Elasticsearch. Default: 1000', - u'action': u'store', - u'required': False, - u'default': 1000})] - - def __init__(self, output_mediator, **kwargs): + def __init__(self, output_mediator): """Initializes the output module object. Args: output_mediator: The output mediator object (instance of OutputMediator). """ - super(TimesketchOutputModule, self).__init__(output_mediator, **kwargs) + super(TimesketchOutputModule, self).__init__(output_mediator) - # Get Elasticsearch config from Timesketch. + self._counter = Counter() + self._doc_type = u'plaso_event' + self._events = [] + self._flush_interval = None + self._index_name = None self._timesketch = timesketch.create_app() + + # TODO: Support reading in server and port information and set this using + # options. with self._timesketch.app_context(): self._elastic_db = ElasticSearchDataStore( host=current_app.config[u'ELASTIC_HOST'], port=current_app.config[u'ELASTIC_PORT']) - self._counter = Counter() - self._doc_type = u'plaso_event' - self._events = [] - self._flush_interval = self._output_mediator.GetConfigurationValue( - u'flush_interval') - self._index_name = self._output_mediator.GetConfigurationValue(u'index') - self._timeline_name = self._output_mediator.GetConfigurationValue(u'name') - hostname = self._output_mediator.GetStoredHostname() if hostname: logging.info(u'Hostname: {0:s}'.format(hostname)) - - # Make sure we have a name for the timeline. Prompt the user if not. - if not self._timeline_name: - if hostname: - self._timeline_name = hostname - else: - # This should not be handles in this module. - # TODO: Move this to CLI code when available. - self._timeline_name = raw_input(u'Timeline name: ') - - logging.info(u'Timeline name: {0:s}'.format(self._timeline_name)) - logging.info(u'Index: {0:s}'.format(self._index_name)) + self._timeline_name = hostname + else: + self._timeline_name = None def _GetSanitizedEventValues(self, event_object): """Builds a dictionary from an event_object. @@ -169,6 +131,44 @@ def Close(self): db_session.add(search_index) db_session.commit() + def GetMissingArguments(self): + """Return a list of arguments that are missing from the input. + + Returns: + a list of argument names that are missing and necessary for the + module to continue to operate. + """ + if not self._timeline_name: + return [u'timeline_name'] + + return [] + + def SetFlushInterval(self, flush_interval): + """Set the flush interval. + + Args: + flush_interval: the flush interval. + """ + self._flush_interval = flush_interval + + def SetIndex(self, index): + """Set the index name. + + Args: + index: the index name. + """ + self._index_name = index + logging.info(u'Index: {0:s}'.format(self._index_name)) + + def SetName(self, name): + """Set the timeline name. + + Args: + name: the timeline name. + """ + self._timeline_name = name + logging.info(u'Timeline name: {0:s}'.format(self._timeline_name)) + def WriteEventBody(self, event_object): """Writes the body of an event object to the output. diff --git a/plaso/output/timesketch_out_test.py b/plaso/output/timesketch_out_test.py index b3259f0e38..508c32f36e 100644 --- a/plaso/output/timesketch_out_test.py +++ b/plaso/output/timesketch_out_test.py @@ -34,9 +34,9 @@ class TimesketchTestConfig(object): """Config object for the tests.""" - name = u'Test' + timeline_name = u'Test' + output_format = u'timesketch' index = u'' - owner = None show_stats = False flush_interval = 1000 @@ -70,8 +70,7 @@ def setUp(self): u'2012-06-27 18:17:01+00:00') self._event_object = TimesketchTestEvent(self._event_timestamp) - test_config = TimesketchTestConfig() - output_mediator = self._CreateOutputMediator(config=test_config) + output_mediator = self._CreateOutputMediator() self._timesketch_output = timesketch_out.TimesketchOutputModule( output_mediator) @@ -100,6 +99,21 @@ def testEventToDict(self): self.assertIsInstance(event_dict, dict) self.assertDictContainsSubset(expected_dict, event_dict) + def testMissingParameters(self): + """Tests the GetMissingArguments function.""" + self.assertListEqual( + self._timesketch_output.GetMissingArguments(), [u'timeline_name']) + + config = TimesketchTestConfig() + + self._timesketch_output.SetIndex(config.index) + self._timesketch_output.SetFlushInterval(config.flush_interval) + self.assertListEqual( + self._timesketch_output.GetMissingArguments(), [u'timeline_name']) + + self._timesketch_output.SetName(config.timeline_name) + self.assertListEqual(self._timesketch_output.GetMissingArguments(), []) + if __name__ == '__main__': unittest.main() diff --git a/plaso/output/tln_test.py b/plaso/output/tln_test.py index a7e0a3cfbb..a1aff4ac20 100644 --- a/plaso/output/tln_test.py +++ b/plaso/output/tln_test.py @@ -17,8 +17,8 @@ def setUp(self): """Sets up the objects needed for this test.""" self._output_writer = cli_test_lib.TestOutputWriter() output_mediator = self._CreateOutputMediator() - self._output_module = tln.TLNOutputModule( - output_mediator, output_writer=self._output_writer) + self._output_module = tln.TLNOutputModule(output_mediator) + self._output_module.SetOutputWriter(self._output_writer) self._event_object = test_lib.TestEventObject() def testWriteHeader(self): @@ -58,8 +58,8 @@ def setUp(self): """Sets up the objects needed for this test.""" output_mediator = self._CreateOutputMediator() self._output_writer = cli_test_lib.TestOutputWriter() - self._output_module = tln.L2TTLNOutputModule( - output_mediator, output_writer=self._output_writer) + self._output_module = tln.L2TTLNOutputModule(output_mediator) + self._output_module.SetOutputWriter(self._output_writer) self._event_object = test_lib.TestEventObject() def testWriteHeader(self): diff --git a/tests/test_extract_and_output.sh b/tests/test_extract_and_output.sh index c122e20c1e..60c62be266 100644 --- a/tests/test_extract_and_output.sh +++ b/tests/test_extract_and_output.sh @@ -204,7 +204,7 @@ do then for CSV_FILE in *.csv; do - mv "${CSV_FILE}" "${TEST_PREFIX}-${CSV_FILE}"; + mv "${CSV_FILE}" "${TEST_PREFIX}-${TEST_OPTIONS_SET}-${CSV_FILE}"; done fi diff --git a/tools/log2timeline.py b/tools/log2timeline.py index eab438538b..641dafca73 100755 --- a/tools/log2timeline.py +++ b/tools/log2timeline.py @@ -220,14 +220,14 @@ def _PrintStatusUpdateStream(self, processing_status): """ if processing_status.GetExtractionCompleted(): self._output_writer.Write( - u'All extraction workers completed - waiting for storage.') + u'All extraction workers completed - waiting for storage.\n') else: for extraction_worker_status in processing_status.extraction_workers: status = extraction_worker_status.status self._output_writer.Write(( u'{0:s} (PID: {1:d}) - events extracted: {2:d} - file: {3:s} ' - u'- running: {4!s} <{5:s}>').format( + u'- running: {4!s} <{5:s}>\n').format( extraction_worker_status.identifier, extraction_worker_status.pid, extraction_worker_status.number_of_events, diff --git a/tools/psort.py b/tools/psort.py index 1a7eb7d822..aeffe37c2c 100755 --- a/tools/psort.py +++ b/tools/psort.py @@ -19,9 +19,12 @@ # TODO: remove after psort options refactor. from plaso.analysis import interface as analysis_interface from plaso.cli import analysis_tool +from plaso.cli import tools as cli_tools from plaso.cli.helpers import manager as helpers_manager +from plaso.frontend import frontend from plaso.frontend import psort from plaso.frontend import utils as frontend_utils +from plaso.output import interface as output_interface from plaso.output import manager as output_manager from plaso.lib import errors from plaso.winnt import language_ids @@ -56,7 +59,6 @@ def __init__(self, input_reader=None, output_writer=None): self._filter_expression = None self._filter_object = None self._front_end = psort.PsortFrontend() - # TODO: remove after psort options refactor. self._options = None self._output_format = None self._time_slice_event_time_string = None @@ -112,11 +114,27 @@ def _ParseFilterOptions(self, options): self._time_slice_duration = getattr(options, u'slice_size', 5) self._use_time_slicer = getattr(options, u'slicer', False) + self._front_end.SetFilter(self._filter_object, self._filter_expression) + # The slice and slicer cannot be set at the same time. if self._time_slice_event_time_string and self._use_time_slicer: raise errors.BadConfigOption( u'Time slice and slicer cannot be used at the same time.') + def _ParseInformationalOptions(self, options): + """Parses the informational options. + + Args: + options: the command line arguments (instance of argparse.Namespace). + + Raises: + BadConfigOption: if the options are invalid. + """ + super(PsortTool, self)._ParseInformationalOptions(options) + + self._quiet_mode = getattr(options, u'quiet', False) + self._front_end.SetQuietMode(self._quiet_mode) + def _ParseLanguageOptions(self, options): """Parses the language options. @@ -124,10 +142,11 @@ def _ParseLanguageOptions(self, options): options: the command line arguments (instance of argparse.Namespace). """ preferred_language = getattr(options, u'preferred_language', u'en-US') + if preferred_language == u'list': self.list_language_identifiers = True else: - self._preferred_language = preferred_language + self._front_end.SetPreferredLanguageIdentifier(preferred_language) def _ProcessStorage(self): """Processes a plaso storage file. @@ -141,12 +160,76 @@ def _ProcessStorage(self): self._time_slice_event_time_string, duration=self._time_slice_duration, timezone=self._timezone) + if self._analysis_plugins: + read_only = False + else: + read_only = True + + try: + storage_file = self._front_end.OpenStorageFile(read_only=read_only) + except IOError as exception: + raise RuntimeError( + u'Unable to open storage file: {0:s} with error: {1:s}.'.format( + self._storage_file_path, exception)) + + output_module = self._front_end.GetOutputModule( + storage_file, preferred_encoding=self.preferred_encoding, + timezone=self._timezone) + + if isinstance(output_module, output_interface.LinearOutputModule): + if self._output_filename: + output_file_object = open(self._output_filename, u'wb') + output_writer = cli_tools.FileObjectOutputWriter(output_file_object) + else: + output_writer = cli_tools.StdoutOutputWriter() + output_module.SetOutputWriter(output_writer) + + # TODO: To set the filter we need to have the filter object. This may + # be better handled in an argument helper, but ATM the argument helper + # does not have access to the actual filter object. + if hasattr(output_module, u'SetFieldsFilter') and self._filter_object: + output_module.SetFieldsFilter(self._filter_object) + + try: + helpers_manager.ArgumentHelperManager.ParseOptions( + self._options, output_module) + except errors.BadConfigOption as exception: + raise RuntimeError(exception) + + # Check if there are parameters that have not been defined and need to + # in order for the output module to continue. Prompt user to supply + # those that may be missing. + missing_parameters = output_module.GetMissingArguments() + while missing_parameters: + configuration_object = frontend.Options() + setattr(configuration_object, u'output_format', output_module.NAME) + for parameter in missing_parameters: + value = self._PromptUserForInput( + u'Missing parameter {0:s} for output module'.format(parameter)) + if value is None: + logging.warning( + u'Unable to set the missing parameter for: {0:s}'.format( + parameter)) + continue + setattr(configuration_object, parameter, value) + helpers_manager.ArgumentHelperManager.ParseOptions( + configuration_object, output_module) + missing_parameters = output_module.GetMissingArguments() + + # Get ANALYSIS PLUGINS AND CONFIGURE! + get_plugins_and_producers = self._front_end.GetAnalysisPluginsAndEventQueues + analysis_plugins, event_queue_producers = get_plugins_and_producers( + self._analysis_plugins) + + for analysis_plugin in analysis_plugins: + helpers_manager.ArgumentHelperManager.ParseOptions( + self._options, analysis_plugin) + counter = self._front_end.ProcessStorage( - self._options, self._analysis_plugins, + output_module, storage_file, analysis_plugins, event_queue_producers, deduplicate_events=self._deduplicate_events, preferred_encoding=self.preferred_encoding, - time_slice=time_slice, timezone=self._timezone, - use_time_slicer=self._use_time_slicer) + time_slice=time_slice, use_time_slicer=self._use_time_slicer) if not self._quiet_mode: logging.info(frontend_utils.FormatHeader(u'Counter')) @@ -154,6 +237,18 @@ def _ProcessStorage(self): # TODO: replace by self._output_writer.Write(). logging.info(frontend_utils.FormatOutputString(element, count)) + def _PromptUserForInput(self, input_text): + """Prompts user for an input and return back read data. + + Args: + input_text: the text used for prompting the user for input. + + Returns: + string containing the input read from the user. + """ + self._output_writer.Write(u'{0:s}: '.format(input_text)) + return self._input_reader.Read() + def AddAnalysisPluginOptions(self, argument_group, plugin_names): """Adds the analysis plugin options to the argument group @@ -233,25 +328,13 @@ def AddOutputModuleOptions(self, argument_group, module_names): Args: argument_group: The argparse argument group (instance of argparse._ArgumentGroup). - module_names: a string containing comma separated output module names. + module_names: a list of output module names. """ - if module_names == u'list': + if u'list' in module_names: return helpers_manager.ArgumentHelperManager.AddCommandLineArguments( - argument_group, u'output') - - # TODO: Remove this once all output modules have been transitioned - # into the CLI argument helpers. - modules_list = set([name.lower() for name in module_names]) - - for name, output_class in self._front_end.GetOutputClasses(): - if not name.lower() in modules_list: - continue - - if output_class.ARGUMENTS: - for parameter, config in output_class.ARGUMENTS: - argument_group.add_argument(parameter, **config) + argument_group, argument_category=u'output', module_list=module_names) def ListAnalysisPlugins(self): """Lists the analysis modules.""" @@ -314,7 +397,8 @@ def ParseArguments(self): level=logging.INFO, format=u'[%(levelname)s] %(message)s') argument_parser = argparse.ArgumentParser( - description=self.DESCRIPTION, add_help=False) + description=self.DESCRIPTION, add_help=False, + conflict_handler=u'resolve') self.AddBasicOptions(argument_parser) self.AddStorageFileOptions(argument_parser) @@ -376,11 +460,14 @@ def ParseArguments(self): argument_index = 0 if argument_index > 0: - module_names = arguments[argument_index] - if module_names == u'list': + module_names_string = arguments[argument_index] + if module_names_string == u'list': self.list_output_modules = True else: - self.AddOutputModuleOptions(output_group, [module_names]) + module_names = module_names_string.split(u',') + module_group = argument_parser.add_argument_group( + u'Output Module Specific Arguments') + self.AddOutputModuleOptions(module_group, module_names) # Add the analysis plugin options. if u'--analysis' in arguments: @@ -471,6 +558,7 @@ def ParseOptions(self, options): self._output_format = getattr(options, u'output_format', None) if not self._output_format: raise errors.BadConfigOption(u'Missing output format.') + self._front_end.SetOutputFormat(self._output_format) if not self._front_end.HasOutputClass(self._output_format): raise errors.BadConfigOption( @@ -479,8 +567,8 @@ def ParseOptions(self, options): self._deduplicate_events = getattr(options, u'dedup', True) self._output_filename = getattr(options, u'write', None) - - self._preferred_language = getattr(options, u'preferred_language', u'en-US') + if self._output_filename: + self._front_end.SetOutputFilename(self._output_filename) if not self._data_location: logging.warning(u'Unable to automatically determine data location.') @@ -514,7 +602,8 @@ def Main(): """The main function.""" multiprocessing.freeze_support() - tool = PsortTool() + input_reader = cli_tools.StdinInputReader() + tool = PsortTool(input_reader=input_reader) if not tool.ParseArguments(): return False diff --git a/tools/psort_test.py b/tools/psort_test.py new file mode 100644 index 0000000000..be910392f2 --- /dev/null +++ b/tools/psort_test.py @@ -0,0 +1,147 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +"""Tests for the psort CLI tool.""" + +import os +import unittest + +from plaso.cli import test_lib as cli_test_lib +from plaso.cli.helpers import interface as helpers_interface +from plaso.cli.helpers import manager as helpers_manager +from plaso.frontend import frontend +from plaso.frontend import psort_test +from plaso.lib import errors +from plaso.output import manager as output_manager +from tools import psort +from tools import test_lib + + +class TestInputReader(object): + """Test input reader.""" + + def __init__(self): + """Initialize the reader.""" + super(TestInputReader, self).__init__() + self.read_called = False + + def Read(self): + """Mock a read operation by user.""" + self.read_called = True + return u'foobar' + + +class TestOutputModuleArgumentHelper(helpers_interface.ArgumentsHelper): + """Test argument helper for the test output module.""" + + NAME = u'test_missing' + + @classmethod + def AddArguments(cls, argument_group): + """Mock the add argument section.""" + pass + + @classmethod + def ParseOptions(cls, options, output_module): + """Provide a test parse options section.""" + if not isinstance(output_module, TestOutputModuleMissingParameters): + raise errors.BadConfigObject(( + u'Output module is not an instance of ' + u'TestOutputModuleMissingParameters')) + + missing = getattr(options, u'missing', None) + if missing: + output_module.SetMissingValue(u'missing', missing) + + parameters = getattr(options, u'parameters', None) + if parameters: + output_module.SetMissingValue(u'parameters', parameters) + + +class TestOutputModuleMissingParameters(psort_test.TestOutputModule): + """Test output module that is missing some parameters.""" + + NAME = u'test_missing' + + # For test purpose assign these as class attributes. + missing = None + parameters = None + + def GetMissingArguments(self): + """Return a list of missing parameters.""" + missing_parameters = [] + if self.missing is None: + missing_parameters.append(u'missing') + + if self.parameters is None: + missing_parameters.append(u'parameters') + + return missing_parameters + + @classmethod + def SetMissingValue(cls, attribute, value): + """Set missing value.""" + setattr(cls, attribute, value) + + +class PsortToolTest(test_lib.ToolTestCase): + """Tests for the psort tool.""" + + def setUp(self): + """Sets up the needed objects used throughout the test.""" + self._input_reader = TestInputReader() + self._output_writer = cli_test_lib.TestOutputWriter(encoding=u'utf-8') + + self._test_tool = psort.PsortTool( + input_reader=self._input_reader, output_writer=self._output_writer) + + def testListOutputModules(self): + """Test the listing of output modules.""" + self._test_tool.ListOutputModules() + raw_data = self._output_writer.ReadOutput() + + # Since the printed output varies depending on which output modules are + # enabled we cannot test the complete string but rather test substrings. + self.assertTrue(raw_data.startswith(( + b'\n******************************** Output Modules ' + b'********************************'))) + + for name, output_class in output_manager.OutputManager.GetOutputClasses(): + expected_string = u'{0:s} : {1:s}'.format(name, output_class.DESCRIPTION) + self.assertTrue(expected_string in raw_data) + + def testProcessStorageWithMissingParameters(self): + """Test the ProcessStorage function with half-configure output module.""" + options = frontend.Options() + options.storage_file = self._GetTestFilePath([u'psort_test.out']) + options.output_format = u'test_missing' + + output_manager.OutputManager.RegisterOutput( + TestOutputModuleMissingParameters) + helpers_manager.ArgumentHelperManager.RegisterHelper( + TestOutputModuleArgumentHelper) + + lines = [] + with cli_test_lib.TempDirectory() as temp_directory: + temp_file_name = os.path.join(temp_directory, u'output.txt') + options.write = temp_file_name + + self._test_tool.ParseOptions(options) + self._test_tool.ProcessStorage() + + with open(temp_file_name, 'rb') as file_object: + for line in file_object: + lines.append(line.strip()) + + self.assertTrue(self._input_reader.read_called) + self.assertEqual(TestOutputModuleMissingParameters.missing, u'foobar') + self.assertEqual(TestOutputModuleMissingParameters.parameters, u'foobar') + + self.assertIn(u'FILE/Unknown FS ctime OS:syslog', lines) + output_manager.OutputManager.DeregisterOutput( + TestOutputModuleMissingParameters) + helpers_manager.ArgumentHelperManager.DeregisterHelper( + TestOutputModuleArgumentHelper) + + +if __name__ == '__main__': + unittest.main()