cypro config fixes

Did a couple of things in this commit. - fixed http_proxy var not loading password - moved crypo init into ace init function - added functionality for unit testing loading password export - moved more bro configs into bro integration config - added database env vars for initial startup (and for the future) - added skip unit tests for bro integration not enabled
unixfreak0037 · Jan 27, 2020 · 60ca603 · 60ca603
1 parent 2cd0d57
commit 60ca603
Show file tree

Hide file tree

Showing 11 changed files with 190 additions and 86 deletions.
diff --git a/ace b/ace
@@ -1378,9 +1378,9 @@ test_parser = subparsers.add_parser('test',
 test_sp = test_parser.add_subparsers(dest='test_cmd')
 
 def test_proxy(args):
-    import requests
     try:
-        requests.get(args.url, proxies=saq.PROXIES)
+        import requests
+        requests.get(args.url, proxies=saq.PROXIES, verify=saq.CONFIG['proxy']['verify'] if 'verify' in saq.CONFIG['proxy'] else False)
         sys.exit(0)
     except Exception as e:
         traceback.print_exc()
@@ -3602,33 +3602,6 @@ if __name__ == '__main__':
     import saq
     saq.initialize(saq_home=saq_home, config_paths=[], logging_config_path=None, args=args, relative_dir=args.relative_dir)
 
-    # has the encryption password been set yet?
-    import saq.crypto
-    from saq.crypto import get_aes_key, InvalidPasswordError
-
-    # are we prompting for the decryption password?
-    if args.provide_decryption_password:
-        while True:
-            saq.ENCRYPTION_PASSWORD_PLAINTEXT = getpass.getpass("Enter the decryption password:")
-            try:
-                saq.ENCRYPTION_PASSWORD = get_aes_key(saq.ENCRYPTION_PASSWORD_PLAINTEXT)
-            except InvalidPasswordError:
-                logging.error("invalid encryption password")
-                continue
-
-            break
-
-    elif saq.crypto.encryption_key_set():
-        # if we're not prompting for it, are we running the encryption cache service yet?
-        logging.debug("reading encryption password from ecs")
-        saq.ENCRYPTION_PASSWORD_PLAINTEXT = saq.crypto.read_ecs()
-        if saq.ENCRYPTION_PASSWORD_PLAINTEXT is not None:
-            try:
-                saq.ENCRYPTION_PASSWORD = get_aes_key(saq.ENCRYPTION_PASSWORD_PLAINTEXT)
-            except InvalidPasswordError:
-                logging.error("read password from ecs but the password is wrong")
-                saq.ENCRYPTION_PASSWORD_PLAINTEXT = None
-
     if args.debug_on_error:
         def info(type, value, tb):
            if hasattr(sys, 'ps1') or not sys.stderr.isatty() or type != AssertionError:

diff --git a/etc/saq.bro.default.ini b/etc/saq.bro.default.ini
@@ -28,3 +28,16 @@ enabled = yes
 
 ; path to the brotex custom whitelist file
 whitelist_path = etc/brotex.whitelist
+
+[module_group_email]
+analysis_module_bro_smtp_analyzer = yes
+
+[analysis_mode_http]
+; mode used for HTTP stream analysis
+module_groups = common, file
+cleanup = yes
+analysis_module_bro_http_analyzer = yes
+
+maximum_cumulative_analysis_warning_time = 30
+maximum_cumulative_analysis_fail_time = 120
+maximum_analysis_time = 20
diff --git a/etc/saq.default.ini b/etc/saq.default.ini
@@ -1892,7 +1892,6 @@ analysis_module_yara_scanner_v3_4 = yes
 
 [module_group_email]
 ; everything related to email scanning
-analysis_module_bro_smtp_analyzer = yes
 analysis_module_email_analyzer = yes
 analysis_module_email_conversation_attachment_analyzer = yes
 analysis_module_email_conversation_frequency_analyzer = yes
@@ -1996,15 +1995,6 @@ cache_dir = cloudphish
 ; cleanup = yes|no
 ; analysis_module_blah = yes|no
 
-[analysis_mode_http]
-; mode used for HTTP stream analysis
-module_groups = common, file
-cleanup = yes
-analysis_module_bro_http_analyzer = yes
-
-maximum_cumulative_analysis_warning_time = 30
-maximum_cumulative_analysis_fail_time = 120
-maximum_analysis_time = 20
 
 [analysis_mode_email]
 ; mode used for email (rfc822 and SMTP stream) analysis

diff --git a/installer/source_install b/installer/source_install
@@ -122,6 +122,7 @@ if grep ACE_DB_USER_PASSWORD etc/saq.ini > /dev/null 2>&1
 then
 	echo "generating mysql accounts for ACE with random password"
 	tr -cd '[:alnum:]' < /dev/urandom | fold -w14 | head -n1 > .mysql.password.sed
+    password=$(cat .mysql.password.sed)
 	# modify the configuration files to use it
 	sed -i -e 's;^;s/ACE_DB_USER_PASSWORD/;' -e 's;$;/g;' .mysql.password.sed
     for path in etc/saq.ini etc/amc_mda.ini etc/amc_client.ini
@@ -133,6 +134,13 @@ then
 	sed -f .mysql.password.sed etc/mysql_defaults.example > etc/mysql_defaults && chmod 660 etc/mysql_defaults 
 	rm .mysql.password.sed
 
+    cat >> load_local_environment <<EOF
+export ACE_DB_NAME="ace"
+export ACE_DB_USER="ace-user"
+export ACE_DB_PASSWORD="$password"
+export ACE_DB_UNIX_SOCKET="/var/run/mysqld/mysqld.sock"
+EOF
+
 	# create the mysql database user for ace
 	sudo mysql < sql/create_db_user.exec.sql && rm sql/create_db_user.exec.sql
 fi

diff --git a/lib/saq/__init__.py b/lib/saq/__init__.py
@@ -1,6 +1,7 @@
 # vim: sw=4:ts=4:et
 
 import datetime
+import json
 import locale
 import logging
 import logging.config
@@ -11,11 +12,12 @@
 import sys
 import time
 import traceback
+import urllib
 
 from getpass import getpass
 
 import ace_api
-from saq.configuration import load_configuration
+from saq.configuration import load_configuration, import_encrypted_passwords
 from saq.constants import *
 from saq.messaging import initialize_message_system
 from saq.network_semaphore import initialize_fallback_semaphores
@@ -147,6 +149,7 @@ def initialize(saq_home=None,
     global DEFAULT_ENCODING
     global DUMP_TRACEBACKS
     global ECS_SOCKET_PATH
+    global ENCRYPTION_INITIALIZED
     global ENCRYPTION_PASSWORD
     global ENCRYPTION_PASSWORD_PLAINTEXT
     global EXCLUDED_SLA_ALERT_TYPES
@@ -197,6 +200,8 @@ def initialize(saq_home=None,
     ENCRYPTION_PASSWORD = None
     # *this* is the password that is used to encrypt/decrypt the ENCRYPTION_PASSWORD at rest
     ENCRYPTION_PASSWORD_PLAINTEXT = None
+    # set to True after we've initialized encryption
+    ENCRYPTION_INITIALIZED = False
 
     # the global log level setting
     LOG_LEVEL = logging.INFO
@@ -356,6 +361,36 @@ def initialize(saq_home=None,
     except Exception as e:
         sys.exit(1)
 
+    # has the encryption password been set yet?
+    import saq.crypto
+    from saq.crypto import get_aes_key, InvalidPasswordError
+
+    if not saq.UNIT_TESTING:
+        # are we prompting for the decryption password?
+        if args and args.provide_decryption_password:
+            while True:
+                ENCRYPTION_PASSWORD_PLAINTEXT = getpass.getpass("Enter the decryption password:")
+                try:
+                    ENCRYPTION_PASSWORD = get_aes_key(ENCRYPTION_PASSWORD_PLAINTEXT)
+                except InvalidPasswordError:
+                    logging.error("invalid encryption password")
+                    continue
+
+                break
+
+        elif saq.crypto.encryption_key_set():
+            # if we're not prompting for it, are we running the encryption cache service yet?
+            logging.debug("reading encryption password from ecs")
+            ENCRYPTION_PASSWORD_PLAINTEXT = saq.crypto.read_ecs()
+            if ENCRYPTION_PASSWORD_PLAINTEXT is not None:
+                try:
+                    ENCRYPTION_PASSWORD = get_aes_key(ENCRYPTION_PASSWORD_PLAINTEXT)
+                except InvalidPasswordError:
+                    logging.error("read password from ecs but the password is wrong")
+                    ENCRYPTION_PASSWORD_PLAINTEXT = None
+
+    ENCRYPTION_INITIALIZED = True
+
     GUI_WHITELIST_EXCLUDED_OBSERVABLE_TYPES = [_.strip() for _ in 
                                                CONFIG['gui']['whitelist_excluded_observable_types'].split(',')]
 
@@ -480,13 +515,16 @@ def initialize(saq_home=None,
             logging.debug("removing proxy environment variable for {}".format(proxy_key))
             del os.environ[proxy_key]
 
-
     # set up the PROXY global dict (to be used with the requests library)
     for proxy_key in [ 'http', 'https' ]:
         if CONFIG['proxy']['host'] and CONFIG['proxy']['port'] and CONFIG['proxy']['transport']:
             if CONFIG['proxy']['user'] and CONFIG['proxy']['password']:
-                PROXIES[proxy_key] = '{}://{}:{}@{}:{}'.format(CONFIG['proxy']['transport'], CONFIG['proxy']['user'], 
-                CONFIG['proxy']['password'], CONFIG['proxy']['host'], CONFIG['proxy']['port'])
+                PROXIES[proxy_key] = '{}://{}:{}@{}:{}'.format(
+                CONFIG['proxy']['transport'], 
+                urllib.parse.quote_plus(CONFIG['proxy']['user']), 
+                urllib.parse.quote_plus(CONFIG['proxy']['password']), 
+                CONFIG['proxy']['host'], 
+                CONFIG['proxy']['port'])
             else:
                 PROXIES[proxy_key] = '{}://{}:{}'.format(CONFIG['proxy']['transport'], CONFIG['proxy']['host'], CONFIG['proxy']['port'])
 
@@ -502,8 +540,11 @@ def initialize(saq_home=None,
                     if 'user' in CONFIG[section] and 'password' in CONFIG[section] \
                     and CONFIG[section]['user'] and CONFIG[section]['password']:
                         OTHER_PROXIES[proxy_name][proxy_key] = '{}://{}:{}@{}:{}'.format(
-                        CONFIG[section]['transport'], CONFIG[section]['user'], CONFIG[section]['password'], 
-                        CONFIG[section]['host'], CONFIG[section]['port'])
+                        CONFIG[section]['transport'], 
+                        urllib.parse.quote_plus(CONFIG[section]['user']), 
+                        urllib.parse.quote_plus(CONFIG[section]['password']), 
+                        CONFIG[section]['host'], 
+                        CONFIG[section]['port'])
                     else:
                         OTHER_PROXIES[proxy_name][proxy_key] = '{}://{}:{}'.format(
                         CONFIG[section]['transport'], CONFIG[section]['host'], CONFIG[section]['port'])

diff --git a/lib/saq/collectors/test_http.py b/lib/saq/collectors/test_http.py
@@ -4,18 +4,23 @@
 import os, os.path
 import shutil
 import tempfile
+import unittest
 
 from subprocess import Popen
 
 import saq
 from saq.collectors.test_bro import BroBaseTestCase
 from saq.collectors.http import BroHTTPStreamCollector
+from saq.integration import integration_enabled
 from saq.test import *
 
 class BroHTTPBaseTestCase(BroBaseTestCase):
     def setUp(self, *args, **kwargs):
         super().setUp(*args, **kwargs)
 
+        if not integration_enabled('bro'):
+            raise unittest.SkipTest("skipping bro tests (bro integration not enabled)")
+
         self.bro_http_dir = os.path.join(saq.DATA_DIR, saq.CONFIG['bro']['http_dir'])
 
         if os.path.exists(self.bro_http_dir):

diff --git a/lib/saq/collectors/test_smtp.py b/lib/saq/collectors/test_smtp.py
@@ -5,21 +5,25 @@
 import re
 import shutil
 import tempfile
+import unittest
 
 from subprocess import Popen
 
 import saq
 from saq.analysis import RootAnalysis
-from saq.constants import *
-from saq.collectors.test_bro import BroBaseTestCase
 from saq.collectors.smtp import BroSMTPStreamCollector
+from saq.collectors.test_bro import BroBaseTestCase
+from saq.constants import *
+from saq.integration import integration_enabled
 from saq.test import *
 from saq.util import storage_dir_from_uuid, workload_storage_dir
-
 class BroSMTPBaseTestCase(BroBaseTestCase):
     def setUp(self, *args, **kwargs):
         super().setUp(*args, **kwargs)
 
+        if not integration_enabled('bro'):
+            raise unittest.SkipTest("skipping bro tests (bro integration not enabled)")
+
         self.bro_smtp_dir = os.path.join(saq.DATA_DIR, saq.CONFIG['bro']['smtp_dir'])
 
         if os.path.exists(self.bro_smtp_dir):

diff --git a/lib/saq/configuration.py b/lib/saq/configuration.py
@@ -4,6 +4,7 @@
 #
 
 import base64
+import json
 import logging
 import os, os.path
 import sys
@@ -30,6 +31,10 @@ def __init__(self, *args, **kwargs):
 
 class EncryptedPasswordInterpolation(Interpolation):
     def before_get(self, parser, section, option, value, defaults):
+        # if we have not initialized encryption yet then just return as-is
+        if not saq.ENCRYPTION_INITIALIZED:
+            return value
+
         # if this is not an encrypted value then just return it as-is
         if value is None or not value.startswith('encrypted:'):
             return value
@@ -72,6 +77,17 @@ def load_configuration():
 
 def _load_configuration():
     default_config = ExtendedConfigParser(allow_no_value=True, interpolation=EncryptedPasswordInterpolation())
+
+    # XXX HACK
+    # optionally when unit testing, the local site passwords can be saved in etc/unittest.passwords.json
+    # this will automatically load these passwords, not requiring ecs running
+    if saq.UNIT_TESTING:
+        unittest_passwords_path = os.path.join(saq.SAQ_HOME, 'etc', 'unittest.passwords.json')
+        if os.path.exists(unittest_passwords_path):
+            logging.info(f"loading passwords from {unittest_passwords_path}")
+            with open(unittest_passwords_path, 'r') as fp:
+                default_config.encrypted_password_cache = json.load(fp)
+
     default_config.read(os.path.join(saq.SAQ_HOME, 'etc', 'saq.default.ini'))
 
     # first we apply the default configuration for integrations