Merge pull request #12271 from Security-Onion-Solutions/suripcap

Suricata PCAP

salt/global/defaults.yaml

@@ -0,0 +1,2 @@
+global:
+  pcapengine: STENO

salt/global/map.jinja

@@ -0,0 +1,2 @@
+{% import_yaml 'global/defaults.yaml' as GLOBALDEFAULTS %}
+{% set GLOBALMERGED = salt['pillar.get']('global', GLOBALDEFAULTS.global, merge=True) %}

salt/global/soc_global.yaml

@@ -10,10 +10,15 @@ global:
     regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$
     regexFailureMessage: You must enter a valid IP address or CIDR.
   mdengine:
-    description: What engine to use for meta data generation. Options are ZEEK and SURICATA.
+    description: Which engine to use for meta data generation. Options are ZEEK and SURICATA.
     regex: ^(ZEEK|SURICATA)$
     regexFailureMessage: You must enter either ZEEK or SURICATA.
     global: True
+  pcapengine:
+    description: Which engine to use for generating pcap. Options are STENO and SURICATA.
+    regex: ^(STENO|SURICATA)$
+    regexFailureMessage: You must enter either STENO or SURICATA.
+    global: True
   ids:
     description: Which IDS engine to use. Currently only Suricata is supported.
     global: True

salt/manager/tools/sbin/so-minion

@@ -79,6 +79,30 @@ function getinstallinfo() {
 	source <(echo $INSTALLVARS)
 }
 
+function pcapspace() {
+	if [[ "$OPERATION" == "setup" ]]; then
+		local SPACESIZE=$(df -k /nsm | tail -1 | awk '{print $2}' | tr -d \n)
+	else
+
+		local NSMSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/nsm"."1K-blocks" ')
+		local ROOTSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/"."1K-blocks" ')
+
+		if [[ "$NSMSIZE" == "null" ]]; then
+			# Looks like there is no dedicated nsm partition. Using root
+			local SPACESIZE=$ROOTSIZE
+		else
+			local SPACESIZE=$NSMSIZE
+		fi
+	fi
+
+	local s=$(( $SPACESIZE / 1000000 ))
+	local s1=$(( $s / 2 ))
+	local s2=$(( $s1 / $lb_procs ))
+
+	MAXPCAPFILES=$s2
+
+}
+
 function testMinion() {
 	# Always run on the host, since this is going to be the manager of a distributed grid, or an eval/standalone.
 	# Distributed managers must run this in order for the sensor nodes to have access to the so-tcpreplay image.
@@ -247,11 +271,17 @@ function add_sensor_to_minion() {
 	echo "  config:" >> $PILLARFILE
 	echo "    af-packet:" >> $PILLARFILE
 	echo "      threads: '$CORECOUNT'" >> $PILLARFILE
+	if [[ $is_pcaplimit ]]; then
+		echo "    output:" >> $PILLARFILE
+		echo "      pcap-log:" >> $PILLARFILE
+		echo "        max-files: '$MAXPCAPFILES'" >> $PILLARFILE
+	fi
 	echo "pcap:" >> $PILLARFILE
 	echo "  enabled: True" >> $PILLARFILE
 	if [[ $is_pcaplimit ]]; then
 		echo "  config:" >> $PILLARFILE
 		echo "    diskfreepercentage: 60" >> $PILLARFILE
+		pcapspace
 	fi
 	echo "  " >> $PILLARFILE
 }
@@ -422,6 +452,7 @@ function updateMine() {
 
 function createEVAL() {
 	is_pcaplimit=true
+	pcapspace
 	add_elasticsearch_to_minion
 	add_sensor_to_minion
 	add_strelka_to_minion
@@ -442,6 +473,7 @@ function createEVAL() {
 
 function createSTANDALONE() {
 	is_pcaplimit=true
+	pcapspace
 	add_elasticsearch_to_minion
 	add_logstash_to_minion
 	add_sensor_to_minion
@@ -531,6 +563,7 @@ function createIDH() {
 
 function createHEAVYNODE() {
 	is_pcaplimit=true
+	pcapspace
 	add_elasticsearch_to_minion
 	add_elastic_agent_to_minion
 	add_logstash_to_minion
@@ -541,6 +574,8 @@ function createHEAVYNODE() {
 }
 
 function createSENSOR() {
+	is_pcaplimit=true
+	pcapspace
 	add_sensor_to_minion
 	add_strelka_to_minion
 	add_telegraf_to_minion

salt/pcap/config.map.jinja

@@ -2,6 +2,12 @@
    or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
    https://securityonion.net/license; you may not use this file except in compliance with the
    Elastic License 2.0. #}
-
+
+{% from 'vars/globals.map.jinja' import GLOBALS %}
 {% import_yaml 'pcap/defaults.yaml' as PCAPDEFAULTS %}
 {% set PCAPMERGED = salt['pillar.get']('pcap', PCAPDEFAULTS.pcap, merge=True) %}
+
+{# disable stenographer if the pcap engine is set to SURICATA #}
+{% if GLOBALS.pcap_engine == "SURICATA" %}
+{%   do PCAPMERGED.update({'enabled': False}) %}
+{% endif %}

salt/sensoroni/enabled.sls

@@ -23,6 +23,10 @@ so-sensoroni:
       - /opt/so/conf/sensoroni/sensoroni.json:/opt/sensoroni/sensoroni.json:ro
       - /opt/so/conf/sensoroni/analyzers:/opt/sensoroni/analyzers:rw
       - /opt/so/log/sensoroni:/opt/sensoroni/logs:rw
+      {% if GLOBALS.pcap_engine == "SURICATA" %}
+      - /nsm/suripcap/:/nsm/suripcap:rw
+      - /nsm/suripcaptmp:/nsm/suripcaptmp:rw
+      {% endif %}
       {% if DOCKER.containers['so-sensoroni'].custom_bind_mounts %}
         {% for BIND in DOCKER.containers['so-sensoroni'].custom_bind_mounts %}
       - {{ BIND }}

salt/sensoroni/files/sensoroni.json

@@ -1,6 +1,7 @@
 {%- from 'vars/globals.map.jinja' import GLOBALS %}
 {%- from 'sensoroni/map.jinja' import SENSORONIMERGED %}
 {%- from 'pcap/config.map.jinja' import PCAPMERGED %}
+{%- from 'suricata/map.jinja' import SURICATAMERGED %}
 {
   "logFilename": "/opt/sensoroni/logs/sensoroni.log",
   "logLevel":"info",
@@ -23,13 +24,22 @@
       "importer": {},
       "statickeyauth": {
         "apiKey": "{{ GLOBALS.sensoroni_key }}"
-{%- if PCAPMERGED.enabled %} 
+{#- if PCAPMERGED.enabled is true then we know that steno is the pcap engine #}
+{#- if it is false, then user has steno disabled in ui or has selected suricata for pcap engine #}
+{%- if PCAPMERGED.enabled %}
       },     
       "stenoquery": {
         "executablePath": "/opt/sensoroni/scripts/stenoquery.sh",
         "pcapInputPath": "/nsm/pcap",
         "pcapOutputPath": "/nsm/pcapout"
       }
+{%- elif GLOBALS.pcap_engine == "SURICATA" and SURICATAMERGED.enabled %}
+      },     
+      "suriquery": {
+        "executablePath": "/opt/sensoroni/scripts/suriquery.sh",
+        "pcapInputPath": "/nsm/suripcap",
+        "pcapOutputPath": "/nsm/pcapout"
+      }
 {%- else %}
       }
 {%- endif %}

salt/suricata/defaults.yaml

@@ -128,6 +128,16 @@ suricata:
         enabled: "no"
       pcap-log:
         enabled: "no"
+        compression: "none"
+        lz4-checksum: "no"
+        lz4-level: 8
+        filename: "%n/so-pcap.%t"
+        limit: "1000mb"
+        mode: "multi"
+        max-files: 10
+        use-stream-depth: "no"
+        conditional: "all"
+        dir: "/nsm/suripcap"
       alert-debug:
         enabled: "no"
       alert-prelude:

salt/suricata/enabled.sls

@@ -12,6 +12,9 @@
 include:
   - suricata.config
   - suricata.sostatus
+{%   if GLOBALS.pcap_engine == "SURICATA" %}
+  - suricata.pcap
+{%   endif %}
 
 so-suricata:
   docker_container.running:
@@ -33,6 +36,9 @@ so-suricata:
       - /nsm/suricata/:/nsm/:rw
       - /nsm/suricata/extracted:/var/log/suricata//filestore:rw
       - /opt/so/conf/suricata/bpf:/etc/suricata/bpf:ro
+      {% if GLOBALS.pcap_engine == "SURICATA" %}
+      - /nsm/suripcap/:/nsm/suripcap:rw
+      {% endif %}
       {% if DOCKER.containers['so-suricata'].custom_bind_mounts %}
         {% for BIND in DOCKER.containers['so-suricata'].custom_bind_mounts %}
       - {{ BIND }}

salt/suricata/map.jinja

@@ -60,6 +60,11 @@
 {%   do SURICATAMERGED.config.outputs['file-store'].update({'enabled':suricata_mdengine.suricata.config.outputs[surimeta_filestore_index]['file-store']['enabled']}) %}
 {% endif %}
 
+{# before we change outputs back to list, enable pcap-log if suricata is the pcapengine #}
+{% if GLOBALS.pcap_engine == "SURICATA" %}
+{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'enabled': 'yes'}) %}
+{% endif %}
+
 {# outputs is a list but we convert to dict in defaults to work with ui #}
 {# below they are converted back to lists #}
 {% load_yaml as outputs %}

salt/suricata/pcap.sls

@@ -0,0 +1,22 @@
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+{% from 'suricata/map.jinja' import SURICATAMERGED %}
+
+suripcapdir:
+  file.directory:
+    - name: /nsm/suripcap
+    - user: 940
+    - group: 939
+    - mode: 755
+    - makedirs: True
+
+{# there should only be 1 interface in af-packet so we can just reference the first list item #}
+{% for i in range(1, SURICATAMERGED.config['af-packet'][0].threads + 1) %}
+
+suripcapthread{{i}}dir:
+  file.directory:
+    - name: /nsm/suripcap/{{i}}
+    - user: 940
+    - group: 939
+    - mode: 755
+
+{% endfor %}

salt/suricata/soc_suricata.yaml

@@ -52,6 +52,7 @@ suricata:
       set-cpu-affinity:
         description: Bind(yes) or unbind(no) management and worker threads to a core or range of cores.
         regex: ^(yes|no)$
+        regexFailureMessage: You must enter either yes or no.
         helpLink: suricata.html
       cpu-affinity:
         management-cpu-set:
@@ -161,6 +162,59 @@ suricata:
               header:
                 description: Header name where the actual IP address will be reported.
                 helpLink: suricata.html
+      pcap-log:
+        enabled:
+          description: This value is ignored by SO. pcapengine in globals takes precidence.
+          readonly: True
+          helpLink: suricata.html
+          advanced: True
+        compression: 
+          description: Enable compression of Suricata PCAP. Currently unsupported
+          advanced: True
+          readonly: True
+          helpLink: suricata.html
+        lz4-checksum: 
+          description: Enable PCAP lz4 checksum. Currently unsupported
+          advanced: True
+          readonly: True
+          helpLink: suricata.html
+        lz4-level: 
+          description: lz4 compression level of PCAP. 0 for no compression 16 for max compression. Currently unsupported
+          advanced: True
+          readonly: True
+          helpLink: suricata.html
+        filename:
+          description: Filename output for Suricata PCAP.
+          advanced: True
+          readonly: True
+          helpLink: suricata.html
+        limit:
+          description: File size limit per thread. To determine max PCAP size multiple threads x max-files x limit.
+          helpLink: suricata.html
+        mode: 
+          description: Suricata PCAP mode. Currently only multi is supported.
+          advanced: True
+          readonly: True
+          helpLink: suricata.html
+        max-files: 
+          description: Max PCAP files per thread. To determine max PCAP size multiple threads x max-files x limit.
+          helpLink: suricata.html
+        use-stream-depth: 
+          description: Set to "no" to ignore the stream depth and capture the entire flow. Set this to "yes" to truncate the flow based on the stream depth. 
+          advanced: True
+          regex: ^(yes|no)$
+          regexFailureMessage: You must enter either yes or no.
+          helpLink: suricata.html
+        conditional: 
+          description: Set to "all" to capture PCAP for all flows. Set to "alert" to capture PCAP just for alerts or set to "tag" to capture PCAP for just tagged rules.
+          regex: ^(all|alert|tag)$
+          regexFailureMessage: You must enter either all, alert or tag.
+          helpLink: suricata.html
+        dir:
+          description: Parent directory to store PCAP.
+          advanced: True
+          readonly: True
+          helpLink: suricata.html
     asn1-max-frames:
       description: Maximum nuber of asn1 frames to decode.
       helpLink: suricata.html
@@ -217,6 +271,9 @@ suricata:
         memcap:
           description: Can be specified in kb,mb,gb.
           helpLink: suricata.html
+        depth:
+          description: Controls how far into a stream that reassembly is done.  
+          helpLink: suricata.html
     host:
       hash-size:
         description: Hash size in bytes.

salt/telegraf/config.sls

@@ -41,6 +41,8 @@ tgraf_sync_script_{{script}}:
     - mode: 770
     - template: jinja
     - source: salt://telegraf/scripts/{{script}}
+    - defaults:
+        GLOBALS: {{ GLOBALS }}
 {% endfor %}
 
 telegraf_sbin:

salt/telegraf/map.jinja

@@ -14,4 +14,11 @@
 {%     do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekloss.sh') %}
 {%     do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekcaptureloss.sh') %}
 {%   endif %}
+
+{%   from 'pcap/config.map.jinja' import PCAPMERGED %}
+{#   PCAPMERGED.enabled is set false in soc ui or if suricata is the pcap engine #}
+{%   if not PCAPMERGED.enabled %}
+{%     do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('stenoloss.sh') %}
+{%   endif %}
+
 {% endif %}

salt/telegraf/scripts/oldpcap.sh

@@ -5,13 +5,17 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-
+{%- if GLOBALS.pcap_engine == "SURICATA" %}
+PCAPLOC=/host/nsm/suripcap
+{%- else %}
+PCAPLOC=/host/nsm/pcap
+{%- endif %}
 
 # if this script isn't already running
 if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
 
     # Get the data
-    OLDPCAP=$(find /host/nsm/pcap -type f -exec stat -c'%n %Z' {} + | sort | grep -v "\." | head -n 1 | awk {'print $2'})
+    OLDPCAP=$(find $PCAPLOC -type f -exec stat -c'%n %Z' {} + | sort | grep -v "/\." | head -n 1 | awk {'print $2'})
     DATE=$(date +%s)
     AGE=$(($DATE - $OLDPCAP))
 

salt/vars/globals.map.jinja

@@ -1,5 +1,6 @@
 {% import 'vars/init.map.jinja' as INIT %}
 {% from 'docker/docker.map.jinja' import DOCKER %}
+{% from 'global/map.jinja' import GLOBALMERGED %}
 
 {% from 'vars/' ~ INIT.GRAINS.role.split('-')[1] ~ '.map.jinja' import ROLE_GLOBALS %} {# role is so-role so we have to split off the 'so' #}
 
@@ -20,6 +21,7 @@
     'influxdb_host': INIT.PILLAR.global.influxdb_host,
     'manager_ip': INIT.PILLAR.global.managerip,
     'md_engine': INIT.PILLAR.global.mdengine,
+    'pcap_engine': GLOBALMERGED.pcapengine,
     'pipeline': INIT.PILLAR.global.pipeline,
     'so_version': INIT.PILLAR.global.soversion,
     'so_docker_gateway': DOCKER.gateway,