working on getting a local docker env

.devcontainer/devcontainer.json

@@ -0,0 +1,43 @@
+// For format details, see https://aka.ms/vscode-remote/devcontainer.json or this file's README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.117.1/containers/docker-existing-docker-compose
+// If you want to run as a non-root user in the container, see .devcontainer/docker-compose.yml.
+{
+	"name": "ACE",
+
+	// Update the 'dockerComposeFile' list if you have more compose files or use different names.
+	// The .devcontainer/docker-compose.yml file contains any overrides you need/want to make.
+	"dockerComposeFile": [
+		"..\\docker-compose-dev.yml"
+	],
+
+	// The 'service' property is the name of the service for the container that VS Code should
+	// use. Update this value and .devcontainer/docker-compose.yml to the real service name.
+	"service": "ace",
+
+	// The optional 'workspaceFolder' property is the path VS Code should open by default when
+	// connected. This is typically a file mount in .devcontainer/docker-compose.yml
+	"workspaceFolder": "/opt/ace",
+
+	// Set *default* container specific settings.json values on container create.
+	"settings": { 
+		"terminal.integrated.shell.linux": null
+	},
+
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": []
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Uncomment the next line if you want start specific services in your Docker Compose config.
+	// "runServices": [],
+
+	// Uncomment the next line if you want to keep your containers running after VS Code shuts down.
+	// "shutdownAction": "none",
+
+	// Uncomment the next line to run commands after the container is created - for example installing git.
+	// "postCreateCommand": "apt-get update && apt-get install -y git",
+
+	// Uncomment to connect as a non-root user. See https://aka.ms/vscode-remote/containers/non-root.
+	// "remoteUser": "vscode"
+}

Dockerfile.ace-base

@@ -0,0 +1,51 @@
+FROM python:3.8-buster
+ENV SAQ_HOME /opt/ace
+ENV SAQ_USER ace
+ENV SAQ_GROUP ace
+ENV TZ UTC
+ENV DEBIAN_FRONTEND noninteractive
+RUN groupadd ace \
+    && useradd -g ace -G sudo -m -s /bin/bash ace \
+    && sed -i -e 's/main$/main contrib non-free/g' /etc/apt/sources.list \
+    && apt -y update \
+    && apt -y install --no-install-recommends apt-utils \
+    && mkdir -p /usr/share/man/man1/ \
+    && apt -y install --no-install-recommends \
+        nmap \
+        libldap2-dev \
+        libsasl2-dev \
+        libffi-dev \
+        libimage-exiftool-perl \
+        p7zip-full \
+        p7zip-rar \
+        unzip \
+        zip \
+        unrar \
+        unace-nonfree \
+        libxml2-dev libxslt1-dev \
+        libyaml-dev \
+        ssdeep \
+        poppler-utils \
+        rng-tools \
+        wireshark-common \
+        build-essential \
+        file \
+        less \
+        nginx \
+        default-jre \
+        bsdmainutils \
+        node-esprima \
+        python2.7 \
+        python-pip \
+    && mkdir /opt/signatures \
+    && chown ace:ace /opt/signatures \
+    && mkdir /opt/ace \
+    && chown ace:ace /opt/ace \
+    && python3 -m pip install pip virtualenv --upgrade \
+    && python2.7 -m pip install officeparser
+SHELL ["/bin/bash", "-c"]
+USER ace
+COPY --chown=ace:ace installer/requirements-3.6.txt /home/ace/python-requirements-3.6.txt
+RUN python3 -m virtualenv --python=python3 /home/ace/venv && source /home/ace/venv/bin/activate && python3 -m pip install -r /home/ace/python-requirements-3.6.txt
+RUN echo 'source /home/ace/venv/bin/activate' >> /home/ace/.bashrc
+RUN echo 'export PATH="$PATH:/opt/ace/bin:/opt/ace"' >> /home/ace/.bashrc

Dockerfile.ace-dev

@@ -0,0 +1,6 @@
+FROM ace-base:latest
+USER root
+RUN apt -y install git
+USER ace
+WORKDIR /opt/ace
+VOLUME [ "/opt/ace", "/opt/ace/data" ]

Dockerfile.ace-prod

@@ -0,0 +1,7 @@
+FROM ace-base:latest
+USER ace
+WORKDIR /opt/ace
+COPY --chown=ace:ace . /opt/ace
+# TODO get rid of this ace-ssl image somehow
+COPY --from=ace-ssl:latest --chown=ace:ace /ssl /opt/ace/ssl
+RUN docker/provision/ace/install

Dockerfile.ssl

@@ -2,4 +2,6 @@ FROM alpine:latest
 RUN apk add bash openssl 
 COPY ssl/ /ssl
 COPY docker/provision/ace/install_ssl_certs.sh .
-RUN ./install_ssl_certs.sh
+RUN ./install_ssl_certs.sh \
+    && tr -cd '[:alnum:]' < /dev/urandom | fold -w14 | head -n1 > mysql.ace-superuser.password \
+    && tr -cd '[:alnum:]' < /dev/urandom | fold -w14 | head -n1 > mysql.ace-user.password

bin/initialize_docker.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+
+import os.path
+import random
+import string
+
+def main(): 
+    user_password = ''.join(random.choices(string.ascii_letters, k=random.randint(23, 32)))
+    target_path = os.path.join('sql', 'templates', 'create_db_user.sql')
+    with open(target_path, 'r') as fp_in:
+        sql = fp_in.read().replace('ACE_DB_USER_PASSWORD', user_password)
+        with open(os.path.join('sql', '70-create-db-user.sql'), 'w') as fp:
+            fp.write(sql)
+
+        print(f"created {target_path}")
+
+    target_path = os.path.join('docker', 'provision', 'ace', 'etc', 'mysql_defaults')
+    with open(target_path, 'w') as fp:
+        fp.write(f"""[client]
+host=localhost
+user=ace-user
+password={user_password}""")
+
+    print(f"created {target_path}")
+
+    admin_password = ''.join(random.choices(string.ascii_letters, k=random.randint(23, 32)))
+    target_path = os.path.join('sql', 'templates', 'create_db_super_user.sql')
+    with open(target_path, 'r') as fp_in:
+        sql = fp_in.read().replace('ACE_SUPERUSER_DB_USER_PASSWORD', admin_password)
+        with open(os.path.join('sql', '71-create-db-super-user.sql'), 'w') as fp:
+            fp.write(sql)
+
+    print(f"created {target_path}")
+
+    target_path = os.path.join('docker', 'provision', 'ace', 'etc', 'mysql_defaults.root')
+    with open(target_path, 'w') as fp:
+        fp.write(f"""[client]
+host=localhost
+user=ace-superuser
+password={admin_password}""")
+
+    print(f"created {target_path}")
+
+    target_path = os.path.join('docker', 'provision', 'ace', 'etc', 'saq.docker.passwords.ini')
+    with open(target_path, 'w') as fp:
+        fp.write(f"""
+[database_ace]
+password = {user_password}
+
+[database_collection]
+password = {user_password}
+
+[database_email_archive]
+password = {user_password}
+
+[database_brocess]
+password = {user_password}
+
+[database_vt_hash_cache]
+password = {user_password}""")
+
+    print(f"created {target_path}")
+
+ if __name__ == '__main__':
+    main()

build_images.ps1

@@ -0,0 +1,5 @@
+docker image build -f Dockerfile.ssl -t ace-ssl:latest .
+docker image build -f Dockerfile.ace-base -t ace-base:latest .
+docker image build -f Dockerfile.ace-dev -t ace-dev:latest .
+docker image build -f Dockerfile.ace-prod -t ace-prod:latest .
+docker image build -f Dockerfile.nginx -t ace-nginx:latest .

debug-docker-dev.ps1

@@ -0,0 +1 @@
+docker run -it -u ace --rm --network ace_default --mount "type=bind,source=$(pwd),target=/opt/ace" ace-dev:latest /bin/bash -il

debug-docker.ps1

@@ -0,0 +1 @@
+docker run -it -u ace --rm --network ace_default --mount source=ace-data,target=/opt/ace/data ace-prod:latest /bin/bash -il

docker-compose-dev.yml

@@ -0,0 +1,50 @@
+version: '3.4'
+services:
+    ace-db:
+        image: mysql:5.7
+        command: mysqld --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_520_ci
+        restart: always
+        environment:
+            MYSQL_ROOT_PASSWORD: 'ace'
+        expose:
+            - '3306'
+        volumes:
+            - ace-db-volume:/var/lib/mysql
+            - ./sql:/docker-entrypoint-initdb.d
+        hostname: ace-db
+
+    ace:
+        build:
+            context: .
+            dockerfile: Dockerfile.ace-dev
+        image: ace-dev:latest
+        depends_on:
+            - ace-db
+        command: /bin/sh -c "while sleep 1000; do :; done"
+        restart: always
+        volumes:
+            - .:/opt/ace
+            - ace-data-volume:/opt/ace/data
+            #- ./aceapi:/opt/ace/aceapi
+            #- ./app:/opt/ace/app
+            #- ./bin:/opt/ace/bin
+            #- ./bro:/opt/ace/bro
+            #- ./cron:/opt/ace/cron
+            #- ./docker:/opt/ace/docker
+            #- ./documentation:/opt/ace/documentation
+            #- ./etc:/opt/ace/etc
+            #- ./hunts:/opt/ace/hunts
+            #- ./installer:/opt/ace/installer
+            #- ./saq:/opt/ace/saq
+            #- ./sql:/opt/ace/sql
+            #- ./ssl:/opt/ace/ssl
+            #- ./test_data:/opt/ace/test_data
+            #- ./tests:/opt/ace/tests
+        hostname: ace
+volumes:
+    ace-data-volume:
+        name: ace-data-dev
+
+    ace-db-volume:
+        name: ace-db-dev
+

docker-compose.yml

@@ -18,7 +18,7 @@ services:
             - ace-ssl
         volumes:
             - ace-db-volume:/var/lib/mysql
-            - ./docker/mounts/mysql/init:/docker-entrypoint-initdb.d
+            - ./sql:/docker-entrypoint-initdb.d
         hostname: ace-db
 
     ace-http:
@@ -36,11 +36,12 @@ services:
         volumes:
             - ace-data-volume:/opt/ace/data
         hostname: ace-http
+
     ace:
         build:
             context: .
-            dockerfile: Dockerfile.ace
-        image: ace:latest
+            dockerfile: Dockerfile.ace-prod
+        image: ace-prod:latest
         depends_on:
             - ace-ssl
             - ace-db

docker/provision/ace/install

@@ -24,15 +24,15 @@ done
 
 # set up environment
 cat >> ~ace/.profile<<EOF
-source venv/bin/activate
+source /home/ace/venv/bin/activate
 source $SAQ_HOME/load_environment
 EOF
 
 cat >> $SAQ_HOME/load_local_environment<<EOF
 export SAQ_ENC=ace
 EOF
 
-source venv/bin/activate
+source /home/ace/venv/bin/activate
 source $SAQ_HOME/load_environment
 
 #cp -a etc/saq.example.ini etc/saq.ini || fail "unable to configure saq.local.ini"
@@ -66,11 +66,11 @@ then
     cp etc/amc_client.example.ini etc/amc_client.ini
 fi
 
-echo "generating random secret key for flask"
-tr -cd '[:alnum:]' < /dev/urandom | fold -w30 | head -n1 > .gui_secret_key.sed
-sed -i -e 's;^;s/ACE_SECRET_KEY/;' -e 's;$;/g;' .gui_secret_key.sed
-sed -i -f .gui_secret_key.sed etc/saq.ini
-rm .gui_secret_key.sed
+#echo "generating random secret key for flask"
+#tr -cd '[:alnum:]' < /dev/urandom | fold -w30 | head -n1 > .gui_secret_key.sed
+#sed -i -e 's;^;s/ACE_SECRET_KEY/;' -e 's;$;/g;' .gui_secret_key.sed
+#sed -i -f .gui_secret_key.sed etc/saq.ini
+#rm .gui_secret_key.sed
 
 # create various directories and files
 # XXX clean this up
@@ -82,17 +82,19 @@ done
 if [ ! -e etc/organization.json ]; then echo '{}' > etc/organization.json; fi
 if [ ! -e etc/local_networks.csv ]; then echo 'Indicator,Indicator_Type' > etc/local_networks.csv; fi
 
-# create our ssl certs
-#./docker/provision/ace/install_ssl_certs.sh
-
-# build the documentation
-#( cd docs && make html )
-
 #activate-global-python-argcomplete
 ln -s /opt/signatures /opt/ace/etc/yara
+cat > etc/saq.ini <<EOF
+[config]
+docker_default = etc/saq.docker.ini
+EOF
 cp etc/saq.docker.ini etc/saq.ini
 cp etc/saq.docker.unittest.ini etc/saq.unittest.ini
 
+cp docker/provision/ace/etc/mysql_defaults etc/
+cp docker/provision/ace/etc/mysql_defaults.root etc/
+cp docker/provision/ace/etc/saq.docker.passwords.ini etc/
+
 for l in api apache
 do
 	if [ ! -e etc/$l\_logging.ini ]

etc/saq.docker.ini

@@ -45,7 +45,15 @@ hostname = ace-db
 unix_socket =
 database = ace
 username = ace-user
-password = qJWht0DkBSVfF7
+;ssl_key = ssl/mysql/client-key.pem
+;ssl_cert = ssl/mysql/client-cert.pem
+;ssl_ca = ssl/mysql/ca-cert.pem
+
+[database_collection]
+hostname = ace-db
+unix_socket =
+database = ace
+username = ace-user
 ;ssl_key = ssl/mysql/client-key.pem
 ;ssl_cert = ssl/mysql/client-cert.pem
 ;ssl_ca = ssl/mysql/ca-cert.pem
@@ -55,7 +63,6 @@ hostname = ace-db
 unix_socket =
 database = brocess
 username = ace-user
-password = qJWht0DkBSVfF7
 
 ; how long do we wait for brocess queries to complete (in seconds)
 ; these queries should complete super fast
@@ -67,17 +74,18 @@ hostname = ace-db
 unix_socket =
 database = email-archive
 username = ace-user
-password = qJWht0DkBSVfF7
 
 [database_vt_hash_cache]
 hostname = ace-db
 unix_socket =
-database = vt_hash_cache
+database = vt-hash-cache
 username = ace-user
-password = qJWht0DkBSVfF7
 
 [analysis_module_email_archiver]
 expiration_days = 3
 
 [node_translation]
 docker = ace,ace-http
+
+[config]
+docker_db = etc/saq.docker.passwords.ini

sql/ace_schema.sql → sql/01-ace.sql

@@ -15,6 +15,10 @@
 /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
 /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
 
+CREATE DATABASE IF NOT EXISTS `ace`;
+ALTER DATABASE `ace` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci;
+USE `ace`;
+
 --
 -- Table structure for table `alerts`
 --