From 762a3bea177a021dc97a1dcf9c1771d9c8bcc359 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 25 Jan 2024 09:59:26 -0500 Subject: [PATCH 001/140] Defaults and Annotations --- salt/suricata/defaults.yaml | 10 +++++++ salt/suricata/soc_suricata.yaml | 50 +++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml index e9e39d40ae..4961ae50a8 100644 --- a/salt/suricata/defaults.yaml +++ b/salt/suricata/defaults.yaml @@ -128,6 +128,16 @@ suricata: enabled: "no" pcap-log: enabled: "no" + compression: "none" + lz4-checksum: "no" + lz4-level: 8 + filename: "%n/so-pcap.%t" + limit: "1000mb" + mode: "multi" + max-files: 10 + use-stream-depth: "no" + conditional: "all" + dir: "/nsm/pcap" alert-debug: enabled: "no" alert-prelude: diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index 30f277c0a7..58a2273b91 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -153,6 +153,53 @@ suricata: header: description: Header name where the actual IP address will be reported. helpLink: suricata.html + pcap-log: + enabled: + description: Enable Suricata to collect PCAP. + helpLink: suricata.html + compression: + description: Enable compression of Suricata PCAP. Currently unsupported + advanced: True + readonly: True + helpLink: suricata.html + lz4-checksum: + description: Enable PCAP lz4 checksum. Currently unsupported + advanced: True + readonly: True + helpLink: suricata.html + lz4-level: + description: lz4 compression level of PCAP. 0 for no compression 16 for max compression. Currently unsupported + advanced: True + readonly: True + helpLink: suricata.html + filename: + description: Filename output for Suricata PCAP. + advanced: True + readonly: True + helpLink: suricata.html + limit: + description: File size limit per thread. To determine max PCAP size multiple threads x max-files x limit. + helpLink: suricata.html + mode: + description: Suricata PCAP mode. Currenlty only multi is supported. + advanced: True + readonly: True + helpLink: suricata.html + max-files: + description: Max PCAP files per thread. To determine max PCAP size multiple threads x max-files x limit. + helpLink: suricata.html + use-stream-depth: + description: Set to "no" to ignore the stream depth and capture the entire flow. Set this to "yes" to truncate the flow based on the stream depth. + advanced: True + helpLink: suricata.html + conditional: + description: Set to "all" to capture PCAP for all flows. Set to "alert" to capture PCAP just for alerts or set to "tag" to capture PCAP for just tagged rules. + helpLink: suricata.html + dir: + description: Parent directory to store PCAP. + advanced: True + readonly: True + helpLink: suricata.html asn1-max-frames: description: Maximum nuber of asn1 frames to decode. helpLink: suricata.html @@ -209,6 +256,9 @@ suricata: memcap: description: Can be specified in kb,mb,gb. helpLink: suricata.html + depth: + description: Controls how far into a stream that reassembly is done. + helpLink: suricata.html host: hash-size: description: Hash size in bytes. From 1a2245a1ed188be34b2ffbcb6304567c9c5abdee Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 29 Jan 2024 13:44:53 -0500 Subject: [PATCH 002/140] Add so-minion modifications --- salt/manager/tools/sbin/so-minion | 25 +++++++++++++++++++++++++ salt/suricata/soc_suricata.yaml | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index d5225cc827..12349b680b 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -79,6 +79,30 @@ function getinstallinfo() { source <(echo $INSTALLVARS) } +function pcapspace() { + + local NSMSIZE=$(salt \* disk.usage --out=json | jq -r '.[]."/nsm"."1K-blocks" ') + local ROOTSIZE=$(salt \* disk.usage --out=json | jq -r '.[]."/"."1K-blocks" ') + + if [[ "$NSMSIZE" == "null" ]]; then + # Looks like there is no dedicated nsm partition. Using root + local SPACESIZE=$ROOTSIZE + else + local SPACESIZE=$NSMSIZE + fi + + local s=$(( $SPACESIZE / 1000000 )) + local s1=$(( $s / 2 )) + local s2=$(( $s1 / $lb_procs )) + + printf '%s\n'\ + "suricata:"\ + " config:"\ + " output:"\ + " pcap-log: $s" >> $PILLARFILE + +} + function testMinion() { # Always run on the host, since this is going to be the manager of a distributed grid, or an eval/standalone. # Distributed managers must run this in order for the sensor nodes to have access to the so-tcpreplay image. @@ -252,6 +276,7 @@ function add_sensor_to_minion() { if [[ $is_pcaplimit ]]; then echo " config:" >> $PILLARFILE echo " diskfreepercentage: 60" >> $PILLARFILE + pcapspace fi echo " " >> $PILLARFILE } diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index 58a2273b91..5dddd7442e 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -181,7 +181,7 @@ suricata: description: File size limit per thread. To determine max PCAP size multiple threads x max-files x limit. helpLink: suricata.html mode: - description: Suricata PCAP mode. Currenlty only multi is supported. + description: Suricata PCAP mode. Currently only multi is supported. advanced: True readonly: True helpLink: suricata.html From 5b05aec96aaa9cbf73294bce24fed2d59f5e4be4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 29 Jan 2024 14:56:51 -0500 Subject: [PATCH 003/140] Target sspecific minion --- salt/manager/tools/sbin/so-minion | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 12349b680b..8777966207 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -81,8 +81,8 @@ function getinstallinfo() { function pcapspace() { - local NSMSIZE=$(salt \* disk.usage --out=json | jq -r '.[]."/nsm"."1K-blocks" ') - local ROOTSIZE=$(salt \* disk.usage --out=json | jq -r '.[]."/"."1K-blocks" ') + local NSMSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/nsm"."1K-blocks" ') + local ROOTSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/"."1K-blocks" ') if [[ "$NSMSIZE" == "null" ]]; then # Looks like there is no dedicated nsm partition. Using root From 0c969312e2ee61eed8e6c0d571e44dc7ee7cdcec Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 29 Jan 2024 15:22:20 -0500 Subject: [PATCH 004/140] Add Globals --- salt/global/defaults.yaml | 2 ++ salt/global/soc_global.yaml | 5 +++++ salt/manager/tools/sbin/so-minion | 3 ++- salt/sensoroni/files/sensoroni.json | 11 ++++++++++- salt/suricata/soc_suricata.yaml | 4 +++- 5 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 salt/global/defaults.yaml diff --git a/salt/global/defaults.yaml b/salt/global/defaults.yaml new file mode 100644 index 0000000000..bd7244a587 --- /dev/null +++ b/salt/global/defaults.yaml @@ -0,0 +1,2 @@ +global: + pcapengine: STENO \ No newline at end of file diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml index 14d637d50e..fc1c09b1ca 100644 --- a/salt/global/soc_global.yaml +++ b/salt/global/soc_global.yaml @@ -14,6 +14,11 @@ global: regex: ^(ZEEK|SURICATA)$ regexFailureMessage: You must enter either ZEEK or SURICATA. global: True + pcapengine: + description: What engine to use for generating pcap. Options are STENO and SURICATA. + regex: ^(STENO|SURICATA)$ + regexFailureMessage: You must enter either STENO or SURICATA. + global: True ids: description: Which IDS engine to use. Currently only Suricata is supported. global: True diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 8777966207..4995e1c9df 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -99,7 +99,8 @@ function pcapspace() { "suricata:"\ " config:"\ " output:"\ - " pcap-log: $s" >> $PILLARFILE + " pcap-log:"\ + " max-files: $s" >> $PILLARFILE } diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index 59ce500e3c..c5608ba567 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -23,13 +23,22 @@ "importer": {}, "statickeyauth": { "apiKey": "{{ GLOBALS.sensoroni_key }}" -{%- if PCAPMERGED.enabled %} +{%- if PCAPMERGED.enabled %} +{%- if PCAPENGINE.steno %} }, "stenoquery": { "executablePath": "/opt/sensoroni/scripts/stenoquery.sh", "pcapInputPath": "/nsm/pcap", "pcapOutputPath": "/nsm/pcapout" } +{%- elif PCAPENGINE.suri %} + }, + "suriquery": { + "executablePath": "/opt/sensoroni/scripts/suriquery.sh", + "pcapInputPath": "/nsm/suripcap", + "pcapOutputPath": "/nsm/pcapout" + } +{%- endif %} {%- else %} } {%- endif %} diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index 5dddd7442e..7153eb9a10 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -155,8 +155,10 @@ suricata: helpLink: suricata.html pcap-log: enabled: - description: Enable Suricata to collect PCAP. + description: This value is ignored by SO. pcapengine in globals takes predidence. + readonly: True helpLink: suricata.html + advanced: True compression: description: Enable compression of Suricata PCAP. Currently unsupported advanced: True From 88c01a22d6454b318c0c94a1718c93fb69b34500 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 29 Jan 2024 15:27:28 -0500 Subject: [PATCH 005/140] Add annotation logic --- salt/suricata/soc_suricata.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index 7153eb9a10..52352d0436 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -44,6 +44,7 @@ suricata: set-cpu-affinity: description: Bind(yes) or unbind(no) management and worker threads to a core or range of cores. regex: ^(yes|no)$ + regexFailureMessage: You must enter either yes or no. helpLink: suricata.html cpu-affinity: management-cpu-set: @@ -155,7 +156,7 @@ suricata: helpLink: suricata.html pcap-log: enabled: - description: This value is ignored by SO. pcapengine in globals takes predidence. + description: This value is ignored by SO. pcapengine in globals takes precidence. readonly: True helpLink: suricata.html advanced: True @@ -193,9 +194,13 @@ suricata: use-stream-depth: description: Set to "no" to ignore the stream depth and capture the entire flow. Set this to "yes" to truncate the flow based on the stream depth. advanced: True + regex: ^(yes|no)$ + regexFailureMessage: You must enter either yes or no. helpLink: suricata.html conditional: description: Set to "all" to capture PCAP for all flows. Set to "alert" to capture PCAP just for alerts or set to "tag" to capture PCAP for just tagged rules. + regex: ^(all|alert|tag)$ + regexFailureMessage: You must enter either all, alert or tag. helpLink: suricata.html dir: description: Parent directory to store PCAP. From ab551a747ddafe1fd0b602e3f84d8130e9ffe5bc Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 29 Jan 2024 15:44:57 -0500 Subject: [PATCH 006/140] Threads placeholder logic --- salt/suricata/enabled.sls | 3 +++ salt/suricata/pcap.sls | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 salt/suricata/pcap.sls diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index ce309e41a4..6dce49c8cc 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -12,6 +12,9 @@ include: - suricata.config - suricata.sostatus + if blah + - suricata.pcap + endif so-suricata: docker_container.running: diff --git a/salt/suricata/pcap.sls b/salt/suricata/pcap.sls new file mode 100644 index 0000000000..f677532f05 --- /dev/null +++ b/salt/suricata/pcap.sls @@ -0,0 +1,25 @@ +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% import_yaml 'suricata/defaults.yaml' as SURICATADEFAULTS %} +{% set SURICATAMERGED = salt['pillar.get']('suricata', SURICATADEFAULTS.suricata, merge=True) %} + +suripcapdir: + file.directory: + - name: /nsm/suripcap + - user: 940 + - group: 939 + - mode: 755 + - makedirs: True + +{{ SURICATAMERGED.config['af-packet'].threads }} + +for thread in afp.threads + +suripcapthreaddir: + file.directory: + - name: /nsm/suripcap/{{thread}} + - user: 940 + - group: 939 + - mode: 755 + - makedirs: True + +endfor \ No newline at end of file From 88d2ddba8bddeeac28fbadf12c826c04a4a61e82 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 29 Jan 2024 15:53:54 -0500 Subject: [PATCH 007/140] add placeholder for telegraf --- salt/telegraf/scripts/oldpcap.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/telegraf/scripts/oldpcap.sh b/salt/telegraf/scripts/oldpcap.sh index bb1be457f7..d3f4b9a93d 100644 --- a/salt/telegraf/scripts/oldpcap.sh +++ b/salt/telegraf/scripts/oldpcap.sh @@ -5,13 +5,18 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. +{%- if pcap is steno +PCAPLOC=/host/nsm/pcap +{%- else %} +PCAPLOC=/host/nsm/suripcap +{%- endif %} # if this script isn't already running if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then # Get the data - OLDPCAP=$(find /host/nsm/pcap -type f -exec stat -c'%n %Z' {} + | sort | grep -v "\." | head -n 1 | awk {'print $2'}) + OLDPCAP=$(find $PCAPLOC -type f -exec stat -c'%n %Z' {} + | sort | grep -v "\." | head -n 1 | awk {'print $2'}) DATE=$(date +%s) AGE=$(($DATE - $OLDPCAP)) From d118ff4728454cb08cd020234b6d858c5edc3656 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 29 Jan 2024 16:54:08 -0500 Subject: [PATCH 008/140] add GLOBALS.pcap_engine --- salt/global/map.jinja | 2 ++ salt/sensoroni/files/sensoroni.json | 6 +++--- salt/suricata/enabled.sls | 4 ++-- salt/suricata/pcap.sls | 13 +++++-------- salt/telegraf/config.sls | 2 ++ salt/telegraf/scripts/oldpcap.sh | 7 +++---- salt/vars/globals.map.jinja | 2 ++ 7 files changed, 19 insertions(+), 17 deletions(-) create mode 100644 salt/global/map.jinja diff --git a/salt/global/map.jinja b/salt/global/map.jinja new file mode 100644 index 0000000000..54abb8c799 --- /dev/null +++ b/salt/global/map.jinja @@ -0,0 +1,2 @@ +{% import_yaml 'global/defaults.yaml' as GLOBALDEFAULTS %} +{% set GLOBALMERGED = salt['pillar.get']('global', GLOBALDEFAULTS.global, merge=True) %} diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index c5608ba567..f813dad2f8 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -24,21 +24,21 @@ "statickeyauth": { "apiKey": "{{ GLOBALS.sensoroni_key }}" {%- if PCAPMERGED.enabled %} -{%- if PCAPENGINE.steno %} +{%- if GLOBALS.pcap_engine == "STENO" %} }, "stenoquery": { "executablePath": "/opt/sensoroni/scripts/stenoquery.sh", "pcapInputPath": "/nsm/pcap", "pcapOutputPath": "/nsm/pcapout" } -{%- elif PCAPENGINE.suri %} +{%- elif GLOBALS.pcap_engine == "SURICATA" %} }, "suriquery": { "executablePath": "/opt/sensoroni/scripts/suriquery.sh", "pcapInputPath": "/nsm/suripcap", "pcapOutputPath": "/nsm/pcapout" } -{%- endif %} +{%- endif %} {%- else %} } {%- endif %} diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index 6dce49c8cc..cf871906bd 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -12,9 +12,9 @@ include: - suricata.config - suricata.sostatus - if blah +{% if GLOBALS.pcap_engine == "SURICATA" %} - suricata.pcap - endif +{% endif %} so-suricata: docker_container.running: diff --git a/salt/suricata/pcap.sls b/salt/suricata/pcap.sls index f677532f05..7a00d8d6a7 100644 --- a/salt/suricata/pcap.sls +++ b/salt/suricata/pcap.sls @@ -1,6 +1,5 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} -{% import_yaml 'suricata/defaults.yaml' as SURICATADEFAULTS %} -{% set SURICATAMERGED = salt['pillar.get']('suricata', SURICATADEFAULTS.suricata, merge=True) %} +{% from 'suricata/map.jinja' import SURICATAMERGED %} suripcapdir: file.directory: @@ -10,16 +9,14 @@ suripcapdir: - mode: 755 - makedirs: True -{{ SURICATAMERGED.config['af-packet'].threads }} -for thread in afp.threads +{% for i in range(1, SURICATAMERGED.config['af-packet'].threads) + 1) %} -suripcapthreaddir: +suripcapthread{{i}}dir: file.directory: - - name: /nsm/suripcap/{{thread}} + - name: /nsm/suripcap/{{i}} - user: 940 - group: 939 - mode: 755 - - makedirs: True -endfor \ No newline at end of file +{% endfor %} diff --git a/salt/telegraf/config.sls b/salt/telegraf/config.sls index 0711260b53..a35be55f51 100644 --- a/salt/telegraf/config.sls +++ b/salt/telegraf/config.sls @@ -41,6 +41,8 @@ tgraf_sync_script_{{script}}: - mode: 770 - template: jinja - source: salt://telegraf/scripts/{{script}} + - defaults: + GLOBALS: {{ GLOBALS }} {% endfor %} telegraf_sbin: diff --git a/salt/telegraf/scripts/oldpcap.sh b/salt/telegraf/scripts/oldpcap.sh index d3f4b9a93d..b68e715392 100644 --- a/salt/telegraf/scripts/oldpcap.sh +++ b/salt/telegraf/scripts/oldpcap.sh @@ -5,13 +5,12 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- if pcap is steno -PCAPLOC=/host/nsm/pcap -{%- else %} +{%- if GLOBALS.pcap_engine == "SURICATA" %} PCAPLOC=/host/nsm/suripcap +{%- else %} +PCAPLOC=/host/nsm/pcap {%- endif %} - # if this script isn't already running if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then diff --git a/salt/vars/globals.map.jinja b/salt/vars/globals.map.jinja index 3265cde183..990aeb39b7 100644 --- a/salt/vars/globals.map.jinja +++ b/salt/vars/globals.map.jinja @@ -1,5 +1,6 @@ {% import 'vars/init.map.jinja' as INIT %} {% from 'docker/docker.map.jinja' import DOCKER %} +{% from 'global/map.jinja' import GLOBALMERGED %} {% from 'vars/' ~ INIT.GRAINS.role.split('-')[1] ~ '.map.jinja' import ROLE_GLOBALS %} {# role is so-role so we have to split off the 'so' #} @@ -20,6 +21,7 @@ 'influxdb_host': INIT.PILLAR.global.influxdb_host, 'manager_ip': INIT.PILLAR.global.managerip, 'md_engine': INIT.PILLAR.global.mdengine, + 'pcap_engine': GLOBALMERGED.pcapengine 'pipeline': INIT.PILLAR.global.pipeline, 'so_version': INIT.PILLAR.global.soversion, 'so_docker_gateway': DOCKER.gateway, From 37dcb84a09d836ec1b772fe25c42051e7bfdf797 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Jan 2024 10:50:01 -0500 Subject: [PATCH 009/140] add missing comma --- salt/vars/globals.map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/vars/globals.map.jinja b/salt/vars/globals.map.jinja index 990aeb39b7..6241732172 100644 --- a/salt/vars/globals.map.jinja +++ b/salt/vars/globals.map.jinja @@ -21,7 +21,7 @@ 'influxdb_host': INIT.PILLAR.global.influxdb_host, 'manager_ip': INIT.PILLAR.global.managerip, 'md_engine': INIT.PILLAR.global.mdengine, - 'pcap_engine': GLOBALMERGED.pcapengine + 'pcap_engine': GLOBALMERGED.pcapengine, 'pipeline': INIT.PILLAR.global.pipeline, 'so_version': INIT.PILLAR.global.soversion, 'so_docker_gateway': DOCKER.gateway, From 0522dc180a753ddb5886c9b370df6fc6662eef10 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Jan 2024 13:39:35 -0500 Subject: [PATCH 010/140] map pcap dir to container. enable pcap-log in map --- salt/suricata/enabled.sls | 3 +++ salt/suricata/map.jinja | 5 +++++ salt/suricata/pcap.sls | 4 ++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index cf871906bd..8b2776b397 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -35,6 +35,9 @@ so-suricata: - /nsm/suricata/:/nsm/:rw - /nsm/suricata/extracted:/var/log/suricata//filestore:rw - /opt/so/conf/suricata/bpf:/etc/suricata/bpf:ro + {% if GLOBALS.pcap_engine == "SURICATA" %} + - /nsm/suripcap/:/nsm/pcap:rw + {% endif %} {% if DOCKER.containers['so-suricata'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-suricata'].custom_bind_mounts %} - {{ BIND }} diff --git a/salt/suricata/map.jinja b/salt/suricata/map.jinja index 01d019de87..5f6e913f50 100644 --- a/salt/suricata/map.jinja +++ b/salt/suricata/map.jinja @@ -60,6 +60,11 @@ {% do SURICATAMERGED.config.outputs['file-store'].update({'enabled':suricata_mdengine.suricata.config.outputs[surimeta_filestore_index]['file-store']['enabled']}) %} {% endif %} +{# before we change outputs back to list, enable pcap-log if suricata is the pcapengine #} +{% if GLOBALS.pcap_engine == "SURICATA" %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'enabled': 'yes'}) %} +{% endif %} + {# outputs is a list but we convert to dict in defaults to work with ui #} {# below they are converted back to lists #} {% load_yaml as outputs %} diff --git a/salt/suricata/pcap.sls b/salt/suricata/pcap.sls index 7a00d8d6a7..a3cbafa0ac 100644 --- a/salt/suricata/pcap.sls +++ b/salt/suricata/pcap.sls @@ -9,8 +9,8 @@ suripcapdir: - mode: 755 - makedirs: True - -{% for i in range(1, SURICATAMERGED.config['af-packet'].threads) + 1) %} +{# there should only be 1 interface in af-packet so we can just reference the first list item #} +{% for i in range(1, SURICATAMERGED.config['af-packet'][0].threads + 1) %} suripcapthread{{i}}dir: file.directory: From 8ed66ea468b5eea0d3230db0e23ea34fd3267762 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Jan 2024 15:22:32 -0500 Subject: [PATCH 011/140] disable stenographer if suricata is pcap engine --- salt/pcap/config.map.jinja | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/pcap/config.map.jinja b/salt/pcap/config.map.jinja index 7ed500f25f..e6d9f8bdac 100644 --- a/salt/pcap/config.map.jinja +++ b/salt/pcap/config.map.jinja @@ -2,6 +2,12 @@ or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at https://securityonion.net/license; you may not use this file except in compliance with the Elastic License 2.0. #} - + +{% from 'vars/globals.map.jinja' import GLOBALS %} {% import_yaml 'pcap/defaults.yaml' as PCAPDEFAULTS %} {% set PCAPMERGED = salt['pillar.get']('pcap', PCAPDEFAULTS.pcap, merge=True) %} + +{# disable stenographer if the pcap engine is set to SURICATA #} +{% if GLOBALS.pcap_engine == "SURICATA" %} +{% do PCAPMERGED.update({'enabled': False}) %} +{% endif %} From f32cb1f1153d691394f58295770717b781929ed9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Jan 2024 15:48:10 -0500 Subject: [PATCH 012/140] fix find to work with steno and suri pcap --- salt/telegraf/scripts/oldpcap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/telegraf/scripts/oldpcap.sh b/salt/telegraf/scripts/oldpcap.sh index b68e715392..438ce912c6 100644 --- a/salt/telegraf/scripts/oldpcap.sh +++ b/salt/telegraf/scripts/oldpcap.sh @@ -15,7 +15,7 @@ PCAPLOC=/host/nsm/pcap if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then # Get the data - OLDPCAP=$(find $PCAPLOC -type f -exec stat -c'%n %Z' {} + | sort | grep -v "\." | head -n 1 | awk {'print $2'}) + OLDPCAP=$(find $PCAPLOC -type f -exec stat -c'%n %Z' {} + | sort | grep -v "/\." | head -n 1 | awk {'print $2'}) DATE=$(date +%s) AGE=$(($DATE - $OLDPCAP)) From 8b503e2ffa722977841947590195b1aae1a90663 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Jan 2024 15:58:11 -0500 Subject: [PATCH 013/140] telegraf dont run stenoloss script if suricata is pcap engine --- salt/telegraf/map.jinja | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/telegraf/map.jinja b/salt/telegraf/map.jinja index e6d3460d67..b56c8a64db 100644 --- a/salt/telegraf/map.jinja +++ b/salt/telegraf/map.jinja @@ -14,4 +14,11 @@ {% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekloss.sh') %} {% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekcaptureloss.sh') %} {% endif %} + +{% from 'pcap/config.map.jinja' import PCAPMERGED %} +{# PCAPMERGED.enabled is set false in soc ui or if suricata is the pcap engine #} +{% if not PCAPMERGED.enabled %} +{% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('stenoloss.sh') %} +{% endif %} + {% endif %} From 8a25748e3309fea6b8f5f01537e4c8f9bdafc65f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Jan 2024 16:06:24 -0500 Subject: [PATCH 014/140] grammar --- salt/global/soc_global.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml index fc1c09b1ca..d707fb1ccc 100644 --- a/salt/global/soc_global.yaml +++ b/salt/global/soc_global.yaml @@ -10,12 +10,12 @@ global: regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$ regexFailureMessage: You must enter a valid IP address or CIDR. mdengine: - description: What engine to use for meta data generation. Options are ZEEK and SURICATA. + description: Which engine to use for meta data generation. Options are ZEEK and SURICATA. regex: ^(ZEEK|SURICATA)$ regexFailureMessage: You must enter either ZEEK or SURICATA. global: True pcapengine: - description: What engine to use for generating pcap. Options are STENO and SURICATA. + description: Which engine to use for generating pcap. Options are STENO and SURICATA. regex: ^(STENO|SURICATA)$ regexFailureMessage: You must enter either STENO or SURICATA. global: True From 0fa4d92f8ff8beb7d8e597bf357ab3a578c04c51 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Thu, 19 Oct 2023 15:49:56 -0600 Subject: [PATCH 015/140] socsigmarepo Need write permissions on the /opt/so/rules dir so I can clone the sigma repo there. --- salt/soc/config.sls | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/soc/config.sls b/salt/soc/config.sls index 902d82ec7f..95135566bf 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -114,6 +114,13 @@ socuploaddir: - group: 939 - makedirs: True +socsigmarepo: + file.directory: + - name: /opt/so/rules + - user: 939 + - group: 939 + - mode: 775 + {% else %} {{sls}}_state_not_allowed: From 4be1214bab11f57286f042ce7dbebd76bcbb8259 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Jan 2024 16:53:57 -0500 Subject: [PATCH 016/140] pcap engine logic for sensoroni --- salt/sensoroni/files/sensoroni.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index f813dad2f8..97c91f0b33 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -1,6 +1,7 @@ {%- from 'vars/globals.map.jinja' import GLOBALS %} {%- from 'sensoroni/map.jinja' import SENSORONIMERGED %} {%- from 'pcap/config.map.jinja' import PCAPMERGED %} +{%- from 'suricata/map.jinja' import SURICATAMERGED %} { "logFilename": "/opt/sensoroni/logs/sensoroni.log", "logLevel":"info", @@ -23,22 +24,22 @@ "importer": {}, "statickeyauth": { "apiKey": "{{ GLOBALS.sensoroni_key }}" +{#- if PCAPMERGED.enabled is true then we know that steno is the pcap engine #} +{#- if it is false, then user has steno disabled in ui or has selected suricata for pcap engine #} {%- if PCAPMERGED.enabled %} -{%- if GLOBALS.pcap_engine == "STENO" %} }, "stenoquery": { "executablePath": "/opt/sensoroni/scripts/stenoquery.sh", "pcapInputPath": "/nsm/pcap", "pcapOutputPath": "/nsm/pcapout" } -{%- elif GLOBALS.pcap_engine == "SURICATA" %} +{%- elif GLOBALS.pcap_engine == "SURICATA" and SURICATAMERGED.enabled %} }, "suriquery": { "executablePath": "/opt/sensoroni/scripts/suriquery.sh", "pcapInputPath": "/nsm/suripcap", "pcapOutputPath": "/nsm/pcapout" } -{%- endif %} {%- else %} } {%- endif %} From 858166bcae280be03c5b416c912fd8c4a7ac8d61 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Tue, 30 Jan 2024 15:43:51 -0700 Subject: [PATCH 017/140] WIP: Detections Changes Removed some strelka/yara rules from salt. Removed yara scripts for downloading and updating rules. This will be managed by SOC. Added a new compile_yara.py script. Added the strelka repos folder. --- salt/manager/init.sls | 53 +------------------ .../manager/tools/sbin_jinja/so-yara-download | 51 ------------------ salt/manager/tools/sbin_jinja/so-yara-update | 41 -------------- salt/soc/files/bin/compile_yara.py | 14 +++++ salt/strelka/backend/config.sls | 10 ---- salt/strelka/config.sls | 9 +++- 6 files changed, 24 insertions(+), 154 deletions(-) delete mode 100644 salt/manager/tools/sbin_jinja/so-yara-download delete mode 100755 salt/manager/tools/sbin_jinja/so-yara-update create mode 100644 salt/soc/files/bin/compile_yara.py diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 23ef189b53..51590a6ec4 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -1,5 +1,5 @@ # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. @@ -61,7 +61,7 @@ manager_sbin: - user: 939 - group: 939 - file_mode: 755 - - exclude_pat: + - exclude_pat: - "*_test.py" yara_update_scripts: @@ -103,55 +103,6 @@ rules_dir: - group: socore - makedirs: True -{% if STRELKAMERGED.rules.enabled %} - -strelkarepos: - file.managed: - - name: /opt/so/conf/strelka/repos.txt - - source: salt://strelka/rules/repos.txt.jinja - - template: jinja - - defaults: - STRELKAREPOS: {{ STRELKAMERGED.rules.repos }} - - makedirs: True - -strelka-yara-update: - {% if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} - cron.present: - {% else %} - cron.absent: - {% endif %} - - user: socore - - name: '/usr/sbin/so-yara-update >> /opt/so/log/yarasync/yara-update.log 2>&1' - - identifier: strelka-yara-update - - hour: '7' - - minute: '1' - -strelka-yara-download: - {% if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} - cron.present: - {% else %} - cron.absent: - {% endif %} - - user: socore - - name: '/usr/sbin/so-yara-download >> /opt/so/log/yarasync/yara-download.log 2>&1' - - identifier: strelka-yara-download - - hour: '7' - - minute: '1' - -{% if not GLOBALS.airgap %} -update_yara_rules: - cmd.run: - - name: /usr/sbin/so-yara-update - - onchanges: - - file: yara_update_scripts - -download_yara_rules: - cmd.run: - - name: /usr/sbin/so-yara-download - - onchanges: - - file: yara_update_scripts -{% endif %} -{% endif %} {% else %} {{sls}}_state_not_allowed: diff --git a/salt/manager/tools/sbin_jinja/so-yara-download b/salt/manager/tools/sbin_jinja/so-yara-download deleted file mode 100644 index aa95762539..0000000000 --- a/salt/manager/tools/sbin_jinja/so-yara-download +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -NOROOT=1 -. /usr/sbin/so-common - -{%- set proxy = salt['pillar.get']('manager:proxy') %} -{%- set noproxy = salt['pillar.get']('manager:no_proxy', '') %} - -# Download the rules from the internet -{%- if proxy %} -export http_proxy={{ proxy }} -export https_proxy={{ proxy }} -export no_proxy="{{ noproxy }}" -{%- endif %} - -repos="/opt/so/conf/strelka/repos.txt" -output_dir=/nsm/rules/yara -gh_status=$(curl -s -o /dev/null -w "%{http_code}" https://github.com) -clone_dir="/tmp" -if [ "$gh_status" == "200" ] || [ "$gh_status" == "301" ]; then - - while IFS= read -r repo; do - if ! $(echo "$repo" | grep -qE '^#'); then - # Remove old repo if existing bc of previous error condition or unexpected disruption - repo_name=`echo $repo | awk -F '/' '{print $NF}'` - [ -d $output_dir/$repo_name ] && rm -rf $output_dir/$repo_name - - # Clone repo and make appropriate directories for rules - git clone $repo $clone_dir/$repo_name - echo "Analyzing rules from $clone_dir/$repo_name..." - mkdir -p $output_dir/$repo_name - # Ensure a copy of the license is available for the rules - [ -f $clone_dir/$repo_name/LICENSE ] && cp $clone_dir/$repo_name/LICENSE $output_dir/$repo_name - - # Copy over rules - for i in $(find $clone_dir/$repo_name -name "*.yar*"); do - rule_name=$(echo $i | awk -F '/' '{print $NF}') - cp $i $output_dir/$repo_name - done - rm -rf $clone_dir/$repo_name - fi - done < $repos - - echo "Done!" - -/usr/sbin/so-yara-update - -else - echo "Server returned $gh_status status code." - echo "No connectivity to Github...exiting..." - exit 1 -fi diff --git a/salt/manager/tools/sbin_jinja/so-yara-update b/salt/manager/tools/sbin_jinja/so-yara-update deleted file mode 100755 index 07c940f47d..0000000000 --- a/salt/manager/tools/sbin_jinja/so-yara-update +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -NOROOT=1 -. /usr/sbin/so-common - -echo "Starting to check for yara rule updates at $(date)..." - -newcounter=0 -excludedcounter=0 -excluded_rules=({{ EXCLUDEDRULES | join(' ') }}) - -# Pull down the SO Rules -SORULEDIR=/nsm/rules/yara -OUTPUTDIR=/opt/so/saltstack/local/salt/strelka/rules - -mkdir -p $OUTPUTDIR -# remove all rules prior to copy so we can clear out old rules -rm -f $OUTPUTDIR/* - -for i in $(find $SORULEDIR -name "*.yar" -o -name "*.yara"); do - rule_name=$(echo $i | awk -F '/' '{print $NF}') - if [[ ! "${excluded_rules[*]}" =~ ${rule_name} ]]; then - echo "Adding rule: $rule_name..." - cp $i $OUTPUTDIR/$rule_name - ((newcounter++)) - else - echo "Excluding rule: $rule_name..." - ((excludedcounter++)) - fi -done - -if [ "$newcounter" -gt 0 ] || [ "$excludedcounter" -gt 0 ];then - echo "$newcounter rules added." - echo "$excludedcounter rule(s) excluded." -fi - -echo "Finished rule updates at $(date)..." diff --git a/salt/soc/files/bin/compile_yara.py b/salt/soc/files/bin/compile_yara.py new file mode 100644 index 0000000000..43c8b1a09c --- /dev/null +++ b/salt/soc/files/bin/compile_yara.py @@ -0,0 +1,14 @@ +import os +import yara +import glob +import sys + +def compile_yara_rules(rules_dir: str) -> None: + compiled_rules_path: str = os.path.join(rules_dir, "rules.yar.compiled") + rule_files: list[str] = glob.glob(os.path.join(rules_dir, '**/*.yar'), recursive=True) + + if rule_files: + rules: yara.Rules = yara.compile(filepaths={os.path.basename(f): f for f in rule_files}) + rules.save(compiled_rules_path) + +compile_yara_rules(sys.argv[1]) diff --git a/salt/strelka/backend/config.sls b/salt/strelka/backend/config.sls index d51debb1b4..b39e06ac83 100644 --- a/salt/strelka/backend/config.sls +++ b/salt/strelka/backend/config.sls @@ -50,16 +50,6 @@ backend_taste: - user: 939 - group: 939 -{% if STRELKAMERGED.rules.enabled %} -strelkarules: - file.recurse: - - name: /opt/so/conf/strelka/rules - - source: salt://strelka/rules - - user: 939 - - group: 939 - - clean: True -{% endif %} - {% else %} {{sls}}_state_not_allowed: diff --git a/salt/strelka/config.sls b/salt/strelka/config.sls index 1d0f75adf4..929bef113c 100644 --- a/salt/strelka/config.sls +++ b/salt/strelka/config.sls @@ -1,5 +1,5 @@ # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. @@ -21,6 +21,13 @@ strelkarulesdir: - group: 939 - makedirs: True +strelkareposdir: + file.directory: + - name: /opt/so/conf/strelka/repos + - user: 939 + - group: 939 + - makedirs: True + strelkadatadir: file.directory: - name: /nsm/strelka From 00289c201ee39e5bba2836f30106dbe7957acf61 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Jan 2024 08:58:57 -0500 Subject: [PATCH 018/140] fix pcap paths --- salt/sensoroni/enabled.sls | 3 +++ salt/suricata/defaults.yaml | 2 +- salt/suricata/enabled.sls | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/sensoroni/enabled.sls b/salt/sensoroni/enabled.sls index 2111e8f1bb..07b4df10a5 100644 --- a/salt/sensoroni/enabled.sls +++ b/salt/sensoroni/enabled.sls @@ -23,6 +23,9 @@ so-sensoroni: - /opt/so/conf/sensoroni/sensoroni.json:/opt/sensoroni/sensoroni.json:ro - /opt/so/conf/sensoroni/analyzers:/opt/sensoroni/analyzers:rw - /opt/so/log/sensoroni:/opt/sensoroni/logs:rw + {% if GLOBALS.pcap_engine == "SURICATA" %} + - /nsm/suripcap/:/nsm/suripcap:rw + {% endif %} {% if DOCKER.containers['so-sensoroni'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-sensoroni'].custom_bind_mounts %} - {{ BIND }} diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml index 4961ae50a8..eb2c181e32 100644 --- a/salt/suricata/defaults.yaml +++ b/salt/suricata/defaults.yaml @@ -137,7 +137,7 @@ suricata: max-files: 10 use-stream-depth: "no" conditional: "all" - dir: "/nsm/pcap" + dir: "/nsm/suripcap" alert-debug: enabled: "no" alert-prelude: diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index 8b2776b397..fa1ebafefa 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -36,7 +36,7 @@ so-suricata: - /nsm/suricata/extracted:/var/log/suricata//filestore:rw - /opt/so/conf/suricata/bpf:/etc/suricata/bpf:ro {% if GLOBALS.pcap_engine == "SURICATA" %} - - /nsm/suripcap/:/nsm/pcap:rw + - /nsm/suripcap/:/nsm/suripcap:rw {% endif %} {% if DOCKER.containers['so-suricata'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-suricata'].custom_bind_mounts %} From 0d01d09d2e8805287cfd061038b1df64ff1348c5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 31 Jan 2024 09:15:35 -0500 Subject: [PATCH 019/140] fix pcap paths --- salt/sensoroni/enabled.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/sensoroni/enabled.sls b/salt/sensoroni/enabled.sls index 07b4df10a5..6dc3df2bd4 100644 --- a/salt/sensoroni/enabled.sls +++ b/salt/sensoroni/enabled.sls @@ -25,6 +25,7 @@ so-sensoroni: - /opt/so/log/sensoroni:/opt/sensoroni/logs:rw {% if GLOBALS.pcap_engine == "SURICATA" %} - /nsm/suripcap/:/nsm/suripcap:rw + - /nsm/suripcaptmp:/nsm/suripcaptmp:rw {% endif %} {% if DOCKER.containers['so-sensoroni'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-sensoroni'].custom_bind_mounts %} From 585147d1de66d700849a23057f7af85c97421433 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Wed, 31 Jan 2024 10:39:47 -0700 Subject: [PATCH 020/140] Added so-detection mapping in elasticsearch --- salt/elasticsearch/defaults.yaml | 31 ++++- .../component/so/detection-mappings.json | 108 ++++++++++++++++++ .../component/so/detection-settings.json | 7 ++ 3 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 salt/elasticsearch/templates/component/so/detection-mappings.json create mode 100644 salt/elasticsearch/templates/component/so/detection-settings.json diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index ce1bfb08de..480467129d 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -117,6 +117,35 @@ elasticsearch: sort: field: '@timestamp' order: desc + so-detection: + index_sorting: false + index_template: + composed_of: + - detection-mappings + - detection-settings + index_patterns: + - so-detection* + priority: 500 + template: + mappings: + date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string + settings: + index: + mapping: + total_fields: + limit: 1500 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc so-common: close: 30 delete: 365 @@ -8909,7 +8938,7 @@ elasticsearch: actions: set_priority: priority: 50 - min_age: 30d + min_age: 30d so-logs-ti_otx_x_threat: index_sorting: false index_template: diff --git a/salt/elasticsearch/templates/component/so/detection-mappings.json b/salt/elasticsearch/templates/component/so/detection-mappings.json new file mode 100644 index 0000000000..df53308f23 --- /dev/null +++ b/salt/elasticsearch/templates/component/so/detection-mappings.json @@ -0,0 +1,108 @@ +{ + "template": { + "mappings": { + "properties": { + "so_audit_doc_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "@timestamp": { + "type": "date" + }, + "so_kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "so_operation": { + "ignore_above": 1024, + "type": "keyword" + }, + "so_detection": { + "properties": { + "publicId": { + "type": "text" + }, + "title": { + "type": "text" + }, + "severity": { + "ignore_above": 1024, + "type": "keyword" + }, + "author": { + "type": "text" + }, + "description": { + "type": "text" + }, + "content": { + "type": "text" + }, + "isEnabled": { + "type": "boolean" + }, + "isReporting": { + "type": "boolean" + }, + "isCommunity": { + "type": "boolean" + }, + "note": { + "type": "text" + }, + "engine": { + "ignore_above": 1024, + "type": "keyword" + }, + "overrides": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "isEnabled": { + "type": "boolean" + }, + "createdAt": { + "type": "date" + }, + "updatedAt": { + "type": "date" + }, + "regex": { + "type": "text" + }, + "value": { + "type": "text" + }, + "thresholdType": { + "ignore_above": 1024, + "type": "keyword" + }, + "track": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "text" + }, + "count": { + "type": "long" + }, + "seconds": { + "type": "long" + }, + "customFilter": { + "type": "text" + } + } + } + } + } + } + } + }, + "_meta": { + "ecs_version": "1.12.2" + } +} \ No newline at end of file diff --git a/salt/elasticsearch/templates/component/so/detection-settings.json b/salt/elasticsearch/templates/component/so/detection-settings.json new file mode 100644 index 0000000000..7b0947a4cf --- /dev/null +++ b/salt/elasticsearch/templates/component/so/detection-settings.json @@ -0,0 +1,7 @@ +{ + "template": {}, + "version": 1, + "_meta": { + "description": "default settings for common Security Onion Detections indices" + } +} \ No newline at end of file From 881d6b313e394a56fc7e64c3de3302a6a1719b3e Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 31 Jan 2024 17:04:11 -0500 Subject: [PATCH 021/140] Update VERSION - kilo --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9cf89c6c7a..7f2e976173 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.50 +2.4.0-kilo From 49b5788ac14d4f79f6cac6b300e5f5cc6357172f Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 1 Feb 2024 07:21:49 -0500 Subject: [PATCH 022/140] add bindings --- salt/soc/enabled.sls | 1 + salt/strelka/backend/enabled.sls | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 2661587f48..0eae0e80e0 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -22,6 +22,7 @@ so-soc: - sobridge: - ipv4_address: {{ DOCKER.containers['so-soc'].ip }} - binds: + - /nsm/rules:/nsm/rules:rw #Need to tighten this up? - /nsm/soc/jobs:/opt/sensoroni/jobs:rw - /nsm/soc/uploads:/nsm/soc/uploads:rw - /opt/so/log/soc/:/opt/sensoroni/logs/:rw diff --git a/salt/strelka/backend/enabled.sls b/salt/strelka/backend/enabled.sls index fc56f41971..9ebb1a1488 100644 --- a/salt/strelka/backend/enabled.sls +++ b/salt/strelka/backend/enabled.sls @@ -42,8 +42,8 @@ strelka_backend: {% endfor %} {% endif %} - restart_policy: on-failure - - watch: - - file: strelkarules + #- watch: + # - file: strelkarules delete_so-strelka-backend_so-status.disabled: file.uncomment: From fe196b56619ee240a81508a111e2e13427e3f382 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 1 Feb 2024 12:22:50 -0500 Subject: [PATCH 023/140] Add SOC Config for Detections --- salt/soc/defaults.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index c1b9470c8f..fdbdfd6b29 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1002,6 +1002,11 @@ soc: jobDir: jobs kratos: hostUrl: + elastalertengine: + communityRulesImportFrequencySeconds: 180 + elastAlertRulesFolder: /opt/so/rules/elastalert + rulesFingerprintFile: /opt/so/conf/soc/sigma.fingerprint + sigmaRulePackages: all elastic: hostUrl: remoteHostUrls: [] @@ -1043,6 +1048,15 @@ soc: - rbac/custom_roles userFiles: - rbac/users_roles + strelkaengine: + compileYaraPythonScriptPath: /opt/so/conf/strelka/compile_yara.py + reposFolder: /nsm/rules/strelka/repos + rulesRepos: + - https://github.com/Security-Onion-Solutions/securityonion-yara + yaraRulesFolder: /opt/so/conf/strelka/rules + suricataengine: + communityRulesFile: /nsm/rules/suricata/emerging-all.rules + rulesFingerprintFile: /opt/so/conf/soc/emerging-all.fingerprint client: enableReverseLookup: false docsUrl: /docs/ From 8f81c9eb68b4c64dace40e065d91cf75b10019f0 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Fri, 2 Feb 2024 11:49:58 -0700 Subject: [PATCH 024/140] Updating config for Detection(s) --- salt/soc/defaults.yaml | 52 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index fdbdfd6b29..7f66864313 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -64,7 +64,7 @@ soc: icon: fa-external-link-alt target: _blank links: - - 'https://{:sublime.url}/messages/{:sublime.message_group_id}' + - 'https://{:sublime.url}/messages/{:sublime.message_group_id}' eventFields: default: - soc_timestamp @@ -1756,3 +1756,53 @@ soc: - amber+strict - red customEnabled: false + detections: + viewEnabled: true + createLink: /detection/create + eventFetchLimit: 500 + eventItemsPerPage: 50 + groupFetchLimit: 50 + mostRecentlyUsedLimit: 5 + safeStringMaxLength: 100 + queryBaseFilter: '_index:"*:so-detection" AND so_kind:detection' + eventFields: + default: + - so_detection.title + - so_detection.isEnabled + - so_detection.engine + - "@timestamp" + queries: + - name: "All Detections" + query: "_id:*" + - name: "Local Rules" + query: "so_detection.isCommunity:false" + - name: "Enabled" + query: "so_detection.isEnabled:true" + - name: "Disabled" + query: "so_detection.isEnabled:false" + - name: "Suricata" + query: "so_detection.engine:suricata" + - name: "ElastAlert" + query: "so_detection.engine:elastalert" + - name: "Strelka" + query: "so_detection.engine:strelka" + detection: + presets: + severity: + customEnabled: false + labels: + - unknown + - informational + - low + - medium + - high + - critical + engine: + customEnabled: false + labels: + - suricata + - elastalert + - strelka + severityTranslations: + minor: low + major: high From 378c99ae8844d78bf6691aa2179cc898e2de17d4 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 2 Feb 2024 18:25:54 -0500 Subject: [PATCH 025/140] Fix bindings --- salt/soc/defaults.yaml | 8 ++++---- salt/soc/enabled.sls | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 7f66864313..6811529bf1 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1004,8 +1004,8 @@ soc: hostUrl: elastalertengine: communityRulesImportFrequencySeconds: 180 - elastAlertRulesFolder: /opt/so/rules/elastalert - rulesFingerprintFile: /opt/so/conf/soc/sigma.fingerprint + elastAlertRulesFolder: /opt/sensoroni/elastalert + rulesFingerprintFile: /opt/sensoroni/fingerprints/sigma.fingerprint sigmaRulePackages: all elastic: hostUrl: @@ -1053,10 +1053,10 @@ soc: reposFolder: /nsm/rules/strelka/repos rulesRepos: - https://github.com/Security-Onion-Solutions/securityonion-yara - yaraRulesFolder: /opt/so/conf/strelka/rules + yaraRulesFolder: /opt/sensoroni/yara suricataengine: communityRulesFile: /nsm/rules/suricata/emerging-all.rules - rulesFingerprintFile: /opt/so/conf/soc/emerging-all.fingerprint + rulesFingerprintFile: /opt/sensoroni/fingerprints/emerging-all.fingerprint client: enableReverseLookup: false docsUrl: /docs/ diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 0eae0e80e0..11f73e7614 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -23,6 +23,9 @@ so-soc: - ipv4_address: {{ DOCKER.containers['so-soc'].ip }} - binds: - /nsm/rules:/nsm/rules:rw #Need to tighten this up? + - /opt/so/rules/yara:/opt/sensoroni/yara:rw + - /opt/so/rules/elastalert/rules:/opt/sensoroni/elastalert:rw + - /opt/so/conf/soc/fingerprints:/opt/sensoroni/fingerprints:rw - /nsm/soc/jobs:/opt/sensoroni/jobs:rw - /nsm/soc/uploads:/nsm/soc/uploads:rw - /opt/so/log/soc/:/opt/sensoroni/logs/:rw From b7b501d289a749c2d8fd0afe246cba9a2cec551b Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 7 Feb 2024 15:02:52 -0500 Subject: [PATCH 026/140] Add Sigma pipelines --- salt/soc/final_sigma_pipeline.yaml | 7 +++++++ salt/soc/so_sigma_pipeline.yaml | 18 ++++++++++++++++++ salt/soc/soc_soc.yaml | 13 +++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 salt/soc/final_sigma_pipeline.yaml create mode 100644 salt/soc/so_sigma_pipeline.yaml diff --git a/salt/soc/final_sigma_pipeline.yaml b/salt/soc/final_sigma_pipeline.yaml new file mode 100644 index 0000000000..656bfbb3ec --- /dev/null +++ b/salt/soc/final_sigma_pipeline.yaml @@ -0,0 +1,7 @@ +name: Security Onion - Final Pipeline +priority: 95 +transformations: + - id: override_field_name_mapping + type: field_name_mapping + mapping: + FieldNameToOverride: NewFieldName diff --git a/salt/soc/so_sigma_pipeline.yaml b/salt/soc/so_sigma_pipeline.yaml new file mode 100644 index 0000000000..a1c4d6d62f --- /dev/null +++ b/salt/soc/so_sigma_pipeline.yaml @@ -0,0 +1,18 @@ +name: Security Onion Baseline Pipeline +priority: 90 +transformations: + - id: baseline_field_name_mapping + type: field_name_mapping + mapping: + cs-method: http.method + c-uri: http.uri + c-useragent: http.useragent + cs-version: http.version + uid: user.uid + sid: rule.uuid + answer: answers + query: dns.query.name + src_ip: destination.ip.keyword + src_port: source.port + dst_ip: destination.ip.keyword + dst_port: destination.port \ No newline at end of file diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 0dd39620b1..fe672fe3e4 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -32,6 +32,14 @@ soc: global: True advanced: True helpLink: soc-customization.html + final_sigma_pipeline__yaml: + title: Final Sigma Pipeline + description: Final Processing Pipeline for Sigma Rules + syntax: yaml + file: True + global: True + advanced: True + helpLink: soc-customization.html config: licenseKey: title: License Key @@ -62,6 +70,11 @@ soc: global: True advanced: True modules: + elastalertengine: + sigmaRulePackages: + description: 'One of the following: core | core+ | core++ | all' + global: True + advanced: False elastic: index: description: Comma-separated list of indices or index patterns (wildcard "*" supported) that SOC will search for records. From 7e3187c0b8f4bba0c992dbf7719a661bc9924735 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 7 Feb 2024 15:35:31 -0500 Subject: [PATCH 027/140] Fixup sigma pipelines --- salt/soc/config.sls | 16 ++++++++++++++++ salt/soc/enabled.sls | 2 ++ .../{ => files/soc}/final_sigma_pipeline.yaml | 0 salt/soc/{ => files/soc}/so_sigma_pipeline.yaml | 0 salt/soc/soc_soc.yaml | 2 +- 5 files changed, 19 insertions(+), 1 deletion(-) rename salt/soc/{ => files/soc}/final_sigma_pipeline.yaml (100%) rename salt/soc/{ => files/soc}/so_sigma_pipeline.yaml (100%) diff --git a/salt/soc/config.sls b/salt/soc/config.sls index 95135566bf..549bf94cfb 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -57,6 +57,22 @@ socmotd: - mode: 600 - template: jinja +socsigmafinalpipeline: + file.managed: + - name: /opt/so/conf/soc/sigma_final_pipeline.yaml + - source: salt://soc/files/soc/sigma_final_pipeline.yaml + - user: 939 + - group: 939 + - mode: 600 + +socsigmasopipeline: + file.managed: + - name: /opt/so/conf/soc/sigma_so_pipeline.yaml + - source: salt://soc/files/soc/sigma_so_pipeline.yaml + - user: 939 + - group: 939 + - mode: 600 + socbanner: file.managed: - name: /opt/so/conf/soc/banner.md diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 11f73e7614..5354231790 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -32,6 +32,8 @@ so-soc: - /opt/so/conf/soc/soc.json:/opt/sensoroni/sensoroni.json:ro - /opt/so/conf/soc/motd.md:/opt/sensoroni/html/motd.md:ro - /opt/so/conf/soc/banner.md:/opt/sensoroni/html/login/banner.md:ro + - /opt/so/conf/soc/sigma_so_pipeline.yaml:/opt/sensoroni/sigma_so_pipeline.yaml:ro + - /opt/so/conf/soc/sigma_final_pipeline.yaml:/opt/sensoroni/sigma_final_pipeline.yaml:rw - /opt/so/conf/soc/custom.js:/opt/sensoroni/html/js/custom.js:ro - /opt/so/conf/soc/custom_roles:/opt/sensoroni/rbac/custom_roles:ro - /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw diff --git a/salt/soc/final_sigma_pipeline.yaml b/salt/soc/files/soc/final_sigma_pipeline.yaml similarity index 100% rename from salt/soc/final_sigma_pipeline.yaml rename to salt/soc/files/soc/final_sigma_pipeline.yaml diff --git a/salt/soc/so_sigma_pipeline.yaml b/salt/soc/files/soc/so_sigma_pipeline.yaml similarity index 100% rename from salt/soc/so_sigma_pipeline.yaml rename to salt/soc/files/soc/so_sigma_pipeline.yaml diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index fe672fe3e4..f413b5c73a 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -32,7 +32,7 @@ soc: global: True advanced: True helpLink: soc-customization.html - final_sigma_pipeline__yaml: + sigma_final_pipeline__yaml: title: Final Sigma Pipeline description: Final Processing Pipeline for Sigma Rules syntax: yaml From 81a3e95914d602386a11e2f5289a98fe59f97ece Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 7 Feb 2024 16:42:16 -0500 Subject: [PATCH 028/140] Fixup sigma pipelines --- .../soc/{final_sigma_pipeline.yaml => sigma_final_pipeline.yaml} | 0 .../files/soc/{so_sigma_pipeline.yaml => sigma_so_pipeline.yaml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename salt/soc/files/soc/{final_sigma_pipeline.yaml => sigma_final_pipeline.yaml} (100%) rename salt/soc/files/soc/{so_sigma_pipeline.yaml => sigma_so_pipeline.yaml} (100%) diff --git a/salt/soc/files/soc/final_sigma_pipeline.yaml b/salt/soc/files/soc/sigma_final_pipeline.yaml similarity index 100% rename from salt/soc/files/soc/final_sigma_pipeline.yaml rename to salt/soc/files/soc/sigma_final_pipeline.yaml diff --git a/salt/soc/files/soc/so_sigma_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml similarity index 100% rename from salt/soc/files/soc/so_sigma_pipeline.yaml rename to salt/soc/files/soc/sigma_so_pipeline.yaml From 29174566f3f7a8307a92eaac2073f108eddedc57 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Thu, 8 Feb 2024 09:44:56 -0700 Subject: [PATCH 029/140] WIP: Updated Detection Mappings, Changed Engine to Language Detection mappings updated to include the removal of Note and the addition of Tags, Ruleset, and Language. SOC defaults updated to use language based queries rather than engine and show the language column instead of the engine column in results. --- .../component/so/detection-mappings.json | 10 +++++++++- salt/soc/defaults.yaml | 18 +++++++++--------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/salt/elasticsearch/templates/component/so/detection-mappings.json b/salt/elasticsearch/templates/component/so/detection-mappings.json index df53308f23..5962367033 100644 --- a/salt/elasticsearch/templates/component/so/detection-mappings.json +++ b/salt/elasticsearch/templates/component/so/detection-mappings.json @@ -47,13 +47,21 @@ "isCommunity": { "type": "boolean" }, - "note": { + "tags": { "type": "text" }, + "ruleset": { + "ignore_above": 1024, + "type": "keyword" + }, "engine": { "ignore_above": 1024, "type": "keyword" }, + "language": { + "ignore_above": 1024, + "type": "keyword" + }, "overrides": { "properties": { "type": { diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 6811529bf1..cc6b417e65 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1769,7 +1769,7 @@ soc: default: - so_detection.title - so_detection.isEnabled - - so_detection.engine + - so_detection.language - "@timestamp" queries: - name: "All Detections" @@ -1781,11 +1781,11 @@ soc: - name: "Disabled" query: "so_detection.isEnabled:false" - name: "Suricata" - query: "so_detection.engine:suricata" - - name: "ElastAlert" - query: "so_detection.engine:elastalert" - - name: "Strelka" - query: "so_detection.engine:strelka" + query: "so_detection.language:suricata" + - name: "Sigma" + query: "so_detection.language:sigma" + - name: "Yara" + query: "so_detection.language:yara" detection: presets: severity: @@ -1797,12 +1797,12 @@ soc: - medium - high - critical - engine: + language: customEnabled: false labels: - suricata - - elastalert - - strelka + - sigma + - yara severityTranslations: minor: low major: high From 64f6d0fba93180f0627b56b5adeb822afa8c5193 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Fri, 9 Feb 2024 14:20:07 -0700 Subject: [PATCH 030/140] Updated Detection's ES Mappings Detection's now have a License field and the Comment model is defined now. --- .../component/so/detection-mappings.json | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/salt/elasticsearch/templates/component/so/detection-mappings.json b/salt/elasticsearch/templates/component/so/detection-mappings.json index 5962367033..4efd2f73f6 100644 --- a/salt/elasticsearch/templates/component/so/detection-mappings.json +++ b/salt/elasticsearch/templates/component/so/detection-mappings.json @@ -62,6 +62,10 @@ "ignore_above": 1024, "type": "keyword" }, + "license": { + "ignore_above": 1024, + "type": "keyword" + }, "overrides": { "properties": { "type": { @@ -106,6 +110,24 @@ } } } + }, + "so_comment": { + "properties": { + "createTime": { + "type": "date" + }, + "detectionId": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "type": "text" + }, + "userId": { + "ignore_above": 1024, + "type": "keyword" + } + } } } } From 5a4e11b2f8f4182330f49ad9b290163e503216c6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 12 Feb 2024 16:09:47 -0500 Subject: [PATCH 031/140] Update soup Remove a function that isn't used any more --- salt/manager/tools/sbin/soup | 61 ------------------------------------ 1 file changed, 61 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 600cb5d4e6..b572610ec6 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -247,67 +247,6 @@ check_sudoers() { fi } -check_log_size_limit() { - local num_minion_pillars - num_minion_pillars=$(find /opt/so/saltstack/local/pillar/minions/ -type f | wc -l) - - if [[ $num_minion_pillars -gt 1 ]]; then - if find /opt/so/saltstack/local/pillar/minions/ -type f | grep -q "_heavynode"; then - lsl_msg='distributed' - fi - else - local minion_id - minion_id=$(lookup_salt_value "id" "" "grains" "" "local") - - local minion_arr - IFS='_' read -ra minion_arr <<< "$minion_id" - - local node_type="${minion_arr[0]}" - - local current_limit - # since it is possible for the salt-master service to be stopped when this is run, we need to check the pillar values locally - # we need to combine default local and default pillars before doing this so we can define --pillar-root in salt-call - local epoch_date=$(date +%s%N) - mkdir -vp /opt/so/saltstack/soup_tmp_${epoch_date}/ - cp -r /opt/so/saltstack/default/pillar/ /opt/so/saltstack/soup_tmp_${epoch_date}/ - # use \cp here to overwrite any pillar files from default with those in local for the tmp directory - \cp -r /opt/so/saltstack/local/pillar/ /opt/so/saltstack/soup_tmp_${epoch_date}/ - current_limit=$(salt-call pillar.get elasticsearch:log_size_limit --local --pillar-root=/opt/so/saltstack/soup_tmp_${epoch_date}/pillar --out=newline_values_only) - rm -rf /opt/so/saltstack/soup_tmp_${epoch_date}/ - - local percent - case $node_type in - 'standalone' | 'eval') - percent=50 - ;; - *) - percent=80 - ;; - esac - - local disk_dir="/" - if [ -d /nsm ]; then - disk_dir="/nsm" - fi - - local disk_size_1k - disk_size_1k=$(df $disk_dir | grep -v "^Filesystem" | awk '{print $2}') - - local ratio="1048576" - - local disk_size_gb - disk_size_gb=$( echo "$disk_size_1k" "$ratio" | awk '{print($1/$2)}' ) - - local new_limit - new_limit=$( echo "$disk_size_gb" "$percent" | awk '{printf("%.0f", $1 * ($2/100))}') - - if [[ $current_limit != "$new_limit" ]]; then - lsl_msg='single-node' - lsl_details=( "$current_limit" "$new_limit" "$minion_id" ) - fi - fi -} - check_os_updates() { # Check to see if there are OS updates echo "Checking for OS updates." From 510226944077b5ac460869d2eced99c0438ba169 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 12 Feb 2024 16:44:54 -0500 Subject: [PATCH 032/140] Update defaults --- salt/idstools/soc_idstools.yaml | 2 +- salt/soc/defaults.yaml | 2 +- salt/soc/soc_soc.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/idstools/soc_idstools.yaml b/salt/idstools/soc_idstools.yaml index 634f68803f..7cf11dba43 100644 --- a/salt/idstools/soc_idstools.yaml +++ b/salt/idstools/soc_idstools.yaml @@ -8,7 +8,7 @@ idstools: global: True helpLink: rules.html ruleset: - description: Defines the ruleset you want to run. Options are ETOPEN or ETPRO. + description: Defines the ruleset you want to run. Options are ETOPEN or ETPRO. -- WARNING -- Changing the ruleset will remove all existing Suricata rules of the previous ruleset and their associated overrides.' global: True regex: ETPRO\b|ETOPEN\b helpLink: rules.html diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index cc6b417e65..fd2eaf8c02 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1050,7 +1050,7 @@ soc: - rbac/users_roles strelkaengine: compileYaraPythonScriptPath: /opt/so/conf/strelka/compile_yara.py - reposFolder: /nsm/rules/strelka/repos + reposFolder: /nsm/rules/yara/repos rulesRepos: - https://github.com/Security-Onion-Solutions/securityonion-yara yaraRulesFolder: /opt/sensoroni/yara diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index f413b5c73a..fe0458820f 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -72,7 +72,7 @@ soc: modules: elastalertengine: sigmaRulePackages: - description: 'One of the following: core | core+ | core++ | all' + description: 'Defines the Sigma Community Ruleset you want to run: core | core+ | core++ | all. -- WARNING -- Changing the ruleset will remove all existing Sigma rules of the previous ruleset and their associated overrides.' global: True advanced: False elastic: From ea80469c2db1bc690e26e4a7e5cf5c1afd44bc3d Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 12 Feb 2024 19:39:55 -0500 Subject: [PATCH 033/140] Detection Default queries --- salt/soc/defaults.yaml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index fd2eaf8c02..29cd7e1ac0 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1770,21 +1770,23 @@ soc: - so_detection.title - so_detection.isEnabled - so_detection.language - - "@timestamp" + - so_detection.severity queries: - name: "All Detections" query: "_id:*" - - name: "Local Rules" + - name: "Custom Detections" query: "so_detection.isCommunity:false" - - name: "Enabled" + - name: "All Detections - Enabled" query: "so_detection.isEnabled:true" - - name: "Disabled" + - name: "All Detections - Disabled" query: "so_detection.isEnabled:false" - - name: "Suricata" + - name: "Detection Type - Suricata (NIDS)" query: "so_detection.language:suricata" - - name: "Sigma" + - name: "Detection Type - Sigma - All" query: "so_detection.language:sigma" - - name: "Yara" + - name: "Detection Type - Sigma - Windows" + query: 'so_detection.language:sigma AND so_detection.content: "*product: windows*"' + - name: "Detection Type - Yara (Strelka)" query: "so_detection.language:yara" detection: presets: From 0c6c6ba2d5d2e529ab6239f53ea6bd38d8a13446 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 13 Feb 2024 13:38:43 -0500 Subject: [PATCH 034/140] Various UI tweaks --- salt/soc/config.sls | 9 ++++++++- salt/soc/defaults.yaml | 6 +++--- salt/soc/enabled.sls | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/salt/soc/config.sls b/salt/soc/config.sls index 549bf94cfb..e4dad8df2e 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -9,9 +9,16 @@ include: - manager.sync_es_users +socdirtest: + file.directory: + - name: /opt/so/rules/elastalert/rules + - user: 939 + - group: 939 + - makedirs: True + socdir: file.directory: - - name: /opt/so/conf/soc + - name: /opt/so/conf/soc/fingerprints - user: 939 - group: 939 - makedirs: True diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 29cd7e1ac0..c060698b4a 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1006,7 +1006,7 @@ soc: communityRulesImportFrequencySeconds: 180 elastAlertRulesFolder: /opt/sensoroni/elastalert rulesFingerprintFile: /opt/sensoroni/fingerprints/sigma.fingerprint - sigmaRulePackages: all + sigmaRulePackages: core elastic: hostUrl: remoteHostUrls: [] @@ -1050,10 +1050,10 @@ soc: - rbac/users_roles strelkaengine: compileYaraPythonScriptPath: /opt/so/conf/strelka/compile_yara.py - reposFolder: /nsm/rules/yara/repos + reposFolder: /opt/sensoroni/yara/repos rulesRepos: - https://github.com/Security-Onion-Solutions/securityonion-yara - yaraRulesFolder: /opt/sensoroni/yara + yaraRulesFolder: /opt/sensoroni/yara/rules suricataengine: communityRulesFile: /nsm/rules/suricata/emerging-all.rules rulesFingerprintFile: /opt/sensoroni/fingerprints/emerging-all.fingerprint diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 5354231790..7c04da8255 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -23,7 +23,7 @@ so-soc: - ipv4_address: {{ DOCKER.containers['so-soc'].ip }} - binds: - /nsm/rules:/nsm/rules:rw #Need to tighten this up? - - /opt/so/rules/yara:/opt/sensoroni/yara:rw + - /opt/so/conf/strelka:/opt/sensoroni/yara:rw - /opt/so/rules/elastalert/rules:/opt/sensoroni/elastalert:rw - /opt/so/conf/soc/fingerprints:/opt/sensoroni/fingerprints:rw - /nsm/soc/jobs:/opt/sensoroni/jobs:rw From 0d297274c8fc4f278e7b9e8530bdd39856425fb9 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Tue, 13 Feb 2024 12:53:18 -0700 Subject: [PATCH 035/140] DetectionComment Mapping Defined --- .../templates/component/so/detection-mappings.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/templates/component/so/detection-mappings.json b/salt/elasticsearch/templates/component/so/detection-mappings.json index 4efd2f73f6..9b68421e71 100644 --- a/salt/elasticsearch/templates/component/so/detection-mappings.json +++ b/salt/elasticsearch/templates/component/so/detection-mappings.json @@ -111,7 +111,7 @@ } } }, - "so_comment": { + "so_detectioncomment": { "properties": { "createTime": { "type": "date" @@ -120,7 +120,7 @@ "ignore_above": 1024, "type": "keyword" }, - "description": { + "value": { "type": "text" }, "userId": { From 031ee078c528b614d398c6420cc5ab1f09bd45ac Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Thu, 19 Oct 2023 15:49:56 -0600 Subject: [PATCH 036/140] socsigmarepo Need write permissions on the /opt/so/rules dir so I can clone the sigma repo there. --- salt/soc/config.sls | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/soc/config.sls b/salt/soc/config.sls index 902d82ec7f..95135566bf 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -114,6 +114,13 @@ socuploaddir: - group: 939 - makedirs: True +socsigmarepo: + file.directory: + - name: /opt/so/rules + - user: 939 + - group: 939 + - mode: 775 + {% else %} {{sls}}_state_not_allowed: From 8800b7e8789297d72637a8a5016f14a1579c8db9 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Tue, 30 Jan 2024 15:43:51 -0700 Subject: [PATCH 037/140] WIP: Detections Changes Removed some strelka/yara rules from salt. Removed yara scripts for downloading and updating rules. This will be managed by SOC. Added a new compile_yara.py script. Added the strelka repos folder. --- salt/manager/init.sls | 53 +------------------ .../manager/tools/sbin_jinja/so-yara-download | 51 ------------------ salt/manager/tools/sbin_jinja/so-yara-update | 41 -------------- salt/soc/files/bin/compile_yara.py | 14 +++++ salt/strelka/backend/config.sls | 10 ---- salt/strelka/config.sls | 9 +++- 6 files changed, 24 insertions(+), 154 deletions(-) delete mode 100644 salt/manager/tools/sbin_jinja/so-yara-download delete mode 100755 salt/manager/tools/sbin_jinja/so-yara-update create mode 100644 salt/soc/files/bin/compile_yara.py diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 23ef189b53..51590a6ec4 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -1,5 +1,5 @@ # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. @@ -61,7 +61,7 @@ manager_sbin: - user: 939 - group: 939 - file_mode: 755 - - exclude_pat: + - exclude_pat: - "*_test.py" yara_update_scripts: @@ -103,55 +103,6 @@ rules_dir: - group: socore - makedirs: True -{% if STRELKAMERGED.rules.enabled %} - -strelkarepos: - file.managed: - - name: /opt/so/conf/strelka/repos.txt - - source: salt://strelka/rules/repos.txt.jinja - - template: jinja - - defaults: - STRELKAREPOS: {{ STRELKAMERGED.rules.repos }} - - makedirs: True - -strelka-yara-update: - {% if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} - cron.present: - {% else %} - cron.absent: - {% endif %} - - user: socore - - name: '/usr/sbin/so-yara-update >> /opt/so/log/yarasync/yara-update.log 2>&1' - - identifier: strelka-yara-update - - hour: '7' - - minute: '1' - -strelka-yara-download: - {% if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} - cron.present: - {% else %} - cron.absent: - {% endif %} - - user: socore - - name: '/usr/sbin/so-yara-download >> /opt/so/log/yarasync/yara-download.log 2>&1' - - identifier: strelka-yara-download - - hour: '7' - - minute: '1' - -{% if not GLOBALS.airgap %} -update_yara_rules: - cmd.run: - - name: /usr/sbin/so-yara-update - - onchanges: - - file: yara_update_scripts - -download_yara_rules: - cmd.run: - - name: /usr/sbin/so-yara-download - - onchanges: - - file: yara_update_scripts -{% endif %} -{% endif %} {% else %} {{sls}}_state_not_allowed: diff --git a/salt/manager/tools/sbin_jinja/so-yara-download b/salt/manager/tools/sbin_jinja/so-yara-download deleted file mode 100644 index aa95762539..0000000000 --- a/salt/manager/tools/sbin_jinja/so-yara-download +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -NOROOT=1 -. /usr/sbin/so-common - -{%- set proxy = salt['pillar.get']('manager:proxy') %} -{%- set noproxy = salt['pillar.get']('manager:no_proxy', '') %} - -# Download the rules from the internet -{%- if proxy %} -export http_proxy={{ proxy }} -export https_proxy={{ proxy }} -export no_proxy="{{ noproxy }}" -{%- endif %} - -repos="/opt/so/conf/strelka/repos.txt" -output_dir=/nsm/rules/yara -gh_status=$(curl -s -o /dev/null -w "%{http_code}" https://github.com) -clone_dir="/tmp" -if [ "$gh_status" == "200" ] || [ "$gh_status" == "301" ]; then - - while IFS= read -r repo; do - if ! $(echo "$repo" | grep -qE '^#'); then - # Remove old repo if existing bc of previous error condition or unexpected disruption - repo_name=`echo $repo | awk -F '/' '{print $NF}'` - [ -d $output_dir/$repo_name ] && rm -rf $output_dir/$repo_name - - # Clone repo and make appropriate directories for rules - git clone $repo $clone_dir/$repo_name - echo "Analyzing rules from $clone_dir/$repo_name..." - mkdir -p $output_dir/$repo_name - # Ensure a copy of the license is available for the rules - [ -f $clone_dir/$repo_name/LICENSE ] && cp $clone_dir/$repo_name/LICENSE $output_dir/$repo_name - - # Copy over rules - for i in $(find $clone_dir/$repo_name -name "*.yar*"); do - rule_name=$(echo $i | awk -F '/' '{print $NF}') - cp $i $output_dir/$repo_name - done - rm -rf $clone_dir/$repo_name - fi - done < $repos - - echo "Done!" - -/usr/sbin/so-yara-update - -else - echo "Server returned $gh_status status code." - echo "No connectivity to Github...exiting..." - exit 1 -fi diff --git a/salt/manager/tools/sbin_jinja/so-yara-update b/salt/manager/tools/sbin_jinja/so-yara-update deleted file mode 100755 index 07c940f47d..0000000000 --- a/salt/manager/tools/sbin_jinja/so-yara-update +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -NOROOT=1 -. /usr/sbin/so-common - -echo "Starting to check for yara rule updates at $(date)..." - -newcounter=0 -excludedcounter=0 -excluded_rules=({{ EXCLUDEDRULES | join(' ') }}) - -# Pull down the SO Rules -SORULEDIR=/nsm/rules/yara -OUTPUTDIR=/opt/so/saltstack/local/salt/strelka/rules - -mkdir -p $OUTPUTDIR -# remove all rules prior to copy so we can clear out old rules -rm -f $OUTPUTDIR/* - -for i in $(find $SORULEDIR -name "*.yar" -o -name "*.yara"); do - rule_name=$(echo $i | awk -F '/' '{print $NF}') - if [[ ! "${excluded_rules[*]}" =~ ${rule_name} ]]; then - echo "Adding rule: $rule_name..." - cp $i $OUTPUTDIR/$rule_name - ((newcounter++)) - else - echo "Excluding rule: $rule_name..." - ((excludedcounter++)) - fi -done - -if [ "$newcounter" -gt 0 ] || [ "$excludedcounter" -gt 0 ];then - echo "$newcounter rules added." - echo "$excludedcounter rule(s) excluded." -fi - -echo "Finished rule updates at $(date)..." diff --git a/salt/soc/files/bin/compile_yara.py b/salt/soc/files/bin/compile_yara.py new file mode 100644 index 0000000000..43c8b1a09c --- /dev/null +++ b/salt/soc/files/bin/compile_yara.py @@ -0,0 +1,14 @@ +import os +import yara +import glob +import sys + +def compile_yara_rules(rules_dir: str) -> None: + compiled_rules_path: str = os.path.join(rules_dir, "rules.yar.compiled") + rule_files: list[str] = glob.glob(os.path.join(rules_dir, '**/*.yar'), recursive=True) + + if rule_files: + rules: yara.Rules = yara.compile(filepaths={os.path.basename(f): f for f in rule_files}) + rules.save(compiled_rules_path) + +compile_yara_rules(sys.argv[1]) diff --git a/salt/strelka/backend/config.sls b/salt/strelka/backend/config.sls index d51debb1b4..b39e06ac83 100644 --- a/salt/strelka/backend/config.sls +++ b/salt/strelka/backend/config.sls @@ -50,16 +50,6 @@ backend_taste: - user: 939 - group: 939 -{% if STRELKAMERGED.rules.enabled %} -strelkarules: - file.recurse: - - name: /opt/so/conf/strelka/rules - - source: salt://strelka/rules - - user: 939 - - group: 939 - - clean: True -{% endif %} - {% else %} {{sls}}_state_not_allowed: diff --git a/salt/strelka/config.sls b/salt/strelka/config.sls index 1d0f75adf4..929bef113c 100644 --- a/salt/strelka/config.sls +++ b/salt/strelka/config.sls @@ -1,5 +1,5 @@ # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. @@ -21,6 +21,13 @@ strelkarulesdir: - group: 939 - makedirs: True +strelkareposdir: + file.directory: + - name: /opt/so/conf/strelka/repos + - user: 939 + - group: 939 + - makedirs: True + strelkadatadir: file.directory: - name: /nsm/strelka From f321e734ebabe5043a01ab79d320abc485d3ef48 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Wed, 31 Jan 2024 10:39:47 -0700 Subject: [PATCH 038/140] Added so-detection mapping in elasticsearch --- salt/elasticsearch/defaults.yaml | 31 ++++- .../component/so/detection-mappings.json | 108 ++++++++++++++++++ .../component/so/detection-settings.json | 7 ++ 3 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 salt/elasticsearch/templates/component/so/detection-mappings.json create mode 100644 salt/elasticsearch/templates/component/so/detection-settings.json diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 03cd6d5190..f4d8c8a95e 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -198,6 +198,35 @@ elasticsearch: sort: field: '@timestamp' order: desc + so-detection: + index_sorting: false + index_template: + composed_of: + - detection-mappings + - detection-settings + index_patterns: + - so-detection* + priority: 500 + template: + mappings: + date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string + settings: + index: + mapping: + total_fields: + limit: 1500 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc so-common: close: 30 delete: 365 @@ -8990,7 +9019,7 @@ elasticsearch: actions: set_priority: priority: 50 - min_age: 30d + min_age: 30d so-logs-ti_otx_x_threat: index_sorting: false index_template: diff --git a/salt/elasticsearch/templates/component/so/detection-mappings.json b/salt/elasticsearch/templates/component/so/detection-mappings.json new file mode 100644 index 0000000000..df53308f23 --- /dev/null +++ b/salt/elasticsearch/templates/component/so/detection-mappings.json @@ -0,0 +1,108 @@ +{ + "template": { + "mappings": { + "properties": { + "so_audit_doc_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "@timestamp": { + "type": "date" + }, + "so_kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "so_operation": { + "ignore_above": 1024, + "type": "keyword" + }, + "so_detection": { + "properties": { + "publicId": { + "type": "text" + }, + "title": { + "type": "text" + }, + "severity": { + "ignore_above": 1024, + "type": "keyword" + }, + "author": { + "type": "text" + }, + "description": { + "type": "text" + }, + "content": { + "type": "text" + }, + "isEnabled": { + "type": "boolean" + }, + "isReporting": { + "type": "boolean" + }, + "isCommunity": { + "type": "boolean" + }, + "note": { + "type": "text" + }, + "engine": { + "ignore_above": 1024, + "type": "keyword" + }, + "overrides": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "isEnabled": { + "type": "boolean" + }, + "createdAt": { + "type": "date" + }, + "updatedAt": { + "type": "date" + }, + "regex": { + "type": "text" + }, + "value": { + "type": "text" + }, + "thresholdType": { + "ignore_above": 1024, + "type": "keyword" + }, + "track": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "text" + }, + "count": { + "type": "long" + }, + "seconds": { + "type": "long" + }, + "customFilter": { + "type": "text" + } + } + } + } + } + } + } + }, + "_meta": { + "ecs_version": "1.12.2" + } +} \ No newline at end of file diff --git a/salt/elasticsearch/templates/component/so/detection-settings.json b/salt/elasticsearch/templates/component/so/detection-settings.json new file mode 100644 index 0000000000..7b0947a4cf --- /dev/null +++ b/salt/elasticsearch/templates/component/so/detection-settings.json @@ -0,0 +1,7 @@ +{ + "template": {}, + "version": 1, + "_meta": { + "description": "default settings for common Security Onion Detections indices" + } +} \ No newline at end of file From c64f37ab671688e46d985806f7267f6c87ce48b0 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Thu, 15 Feb 2024 10:34:07 -0700 Subject: [PATCH 039/140] sigmaRulePackages is now a string array --- salt/soc/defaults.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index a19fb45f04..546114b9f1 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -64,7 +64,7 @@ soc: icon: fa-external-link-alt target: _blank links: - - 'https://{:sublime.url}/messages/{:sublime.message_group_id}' + - 'https://{:sublime.url}/messages/{:sublime.message_group_id}' - name: actionProcessAncestors description: actionProcessAncestorsHelp icon: fa-people-roof @@ -1012,7 +1012,8 @@ soc: communityRulesImportFrequencySeconds: 180 elastAlertRulesFolder: /opt/sensoroni/elastalert rulesFingerprintFile: /opt/sensoroni/fingerprints/sigma.fingerprint - sigmaRulePackages: core + sigmaRulePackages: + - core elastic: hostUrl: remoteHostUrls: [] From ffb3cc87b7adf47452ced95ae0d8aff370a1ef12 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 16 Feb 2024 11:55:10 -0500 Subject: [PATCH 040/140] Default ruleset; Descriptions --- salt/idstools/soc_idstools.yaml | 2 +- salt/soc/defaults.yaml | 1 + salt/soc/soc_soc.yaml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/idstools/soc_idstools.yaml b/salt/idstools/soc_idstools.yaml index 7cf11dba43..f8ec3b8b67 100644 --- a/salt/idstools/soc_idstools.yaml +++ b/salt/idstools/soc_idstools.yaml @@ -8,7 +8,7 @@ idstools: global: True helpLink: rules.html ruleset: - description: Defines the ruleset you want to run. Options are ETOPEN or ETPRO. -- WARNING -- Changing the ruleset will remove all existing Suricata rules of the previous ruleset and their associated overrides.' + description: 'Defines the ruleset you want to run. Options are ETOPEN or ETPRO. WARNING! Changing the ruleset will remove all existing Suricata rules of the previous ruleset and their associated overrides. This removal cannot be undone.' global: True regex: ETPRO\b|ETOPEN\b helpLink: rules.html diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 546114b9f1..91d47cf916 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1014,6 +1014,7 @@ soc: rulesFingerprintFile: /opt/sensoroni/fingerprints/sigma.fingerprint sigmaRulePackages: - core + - emerging_threats_addon elastic: hostUrl: remoteHostUrls: [] diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index fe0458820f..fdfb09733a 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -72,7 +72,7 @@ soc: modules: elastalertengine: sigmaRulePackages: - description: 'Defines the Sigma Community Ruleset you want to run: core | core+ | core++ | all. -- WARNING -- Changing the ruleset will remove all existing Sigma rules of the previous ruleset and their associated overrides.' + description: 'Defines the Sigma Community Ruleset you want to run. One of these (core | core+ | core++ | all ) as well as an optional Add-on (emerging_threats_addon). WARNING! Changing the ruleset will remove all existing Sigma rules of the previous ruleset and their associated overrides. This removal cannot be undone.' global: True advanced: False elastic: From 07fcfab7ec2392db3ecdead036d9deafb9edb6eb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 20 Feb 2024 10:14:11 -0500 Subject: [PATCH 041/140] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9cf89c6c7a..5a99ed019d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.50 +2.4.60 From 4b314c871573144b89bf47fb45eebfc10bb3ba56 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 20 Feb 2024 10:30:09 -0500 Subject: [PATCH 042/140] replace correlate icon to avoid confusion with searcheng.in --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 244a021d38..dc836a9d77 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -20,7 +20,7 @@ soc: - dashboards - name: actionCorrelate description: actionCorrelateHelp - icon: fab fa-searchengin + icon: fa-magnifying-glass-arrow-right target: '' links: - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}" OR "{:network.community_id}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source_geo.organization_name source.geo.country_name | groupby destination_geo.organization_name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid' From 6c6a362fcc14ae2c7df4c71ea4d2299072779d8a Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 20 Feb 2024 19:14:18 -0500 Subject: [PATCH 043/140] add lock threads --- .github/workflows/lock-threads.yml | 42 ++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/lock-threads.yml diff --git a/.github/workflows/lock-threads.yml b/.github/workflows/lock-threads.yml new file mode 100644 index 0000000000..25e5d8c170 --- /dev/null +++ b/.github/workflows/lock-threads.yml @@ -0,0 +1,42 @@ +name: 'Lock Threads' + +on: + schedule: + - cron: '50 1 * * *' + workflow_dispatch: + +permissions: + issues: write + pull-requests: write + discussions: write + +concurrency: + group: lock-threads + +jobs: + close-threads: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/stale@v5 + with: + days-before-issue-stale: -1 + days-before-issue-close: 60 + stale-issue-message: "This issue is stale because it has been inactive for an extended period. Stale issues convey that the issue, while important to someone, is not critical enough for the author, or other community members to work on, sponsor, or otherwise shepherd the issue through to a resolution." + close-issue-message: "This issue was closed because it has been stale for an extended period. It will be automatically locked in 30 days, after which no further commenting will be available." + days-before-pr-stale: 45 + days-before-pr-close: 60 + stale-pr-message: "This PR is stale because it has been inactive for an extended period. The longer a PR remains stale the more out of date with the main branch it becomes." + close-pr-message: "This PR was closed because it has been stale for an extended period. It will be automatically locked in 30 days. If there is still a commitment to finishing this PR re-open it before it is locked." + + lock-threads: + runs-on: ubuntu-latest + steps: + - uses: jertel/lock-threads@main + with: + include-discussion-currently-open: true + discussion-inactive-days: 90 + issue-inactive-days: 30 + pr-inactive-days: 30 From 9ca0f586ae63a90fef9c78b8b1fc24b020545ae3 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 21 Feb 2024 11:45:02 -0500 Subject: [PATCH 044/140] Manage the repos --- salt/manager/files/mirror.txt | 0 salt/manager/files/repodownload.conf | 13 +++++++++++++ salt/manager/init.sls | 14 ++++++++++++++ setup/so-functions | 4 ++-- 4 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 salt/manager/files/mirror.txt create mode 100644 salt/manager/files/repodownload.conf diff --git a/salt/manager/files/mirror.txt b/salt/manager/files/mirror.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/salt/manager/files/repodownload.conf b/salt/manager/files/repodownload.conf new file mode 100644 index 0000000000..3c156a9db6 --- /dev/null +++ b/salt/manager/files/repodownload.conf @@ -0,0 +1,13 @@ +[main] +gpgcheck=1 +installonly_limit=3 +clean_requirements_on_remove=True +best=True +skip_if_unavailable=False +cachedir=/opt/so/conf/reposync/cache +keepcache=0 +[securityonionsync] +name=Security Onion Repo repo +mirrorlist=file:///opt/so/conf/reposync/mirror.txt +enabled=1 +gpgcheck=1 \ No newline at end of file diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 23ef189b53..e51a448d56 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -75,6 +75,20 @@ yara_update_scripts: - defaults: EXCLUDEDRULES: {{ STRELKAMERGED.rules.excluded }} +so-repo-file: + file.managed: + - name: /opt/so/conf/reposync/repodownload.conf + - source: salt://manager/files/repodownload.conf + - user: socore + - group: socore + +so-repo-mirrorlist: + file.managed: + - name: /opt/so/conf/reposync/mirror.txt + - source: salt://manager/files/mirror.txt + - user: socore + - group: socore + so-repo-sync: {% if MANAGERMERGED.reposync.enabled %} cron.present: diff --git a/setup/so-functions b/setup/so-functions index f0462e4d64..ef1df4a719 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1811,8 +1811,8 @@ repo_sync_local() { info "Adding Repo Download Configuration" mkdir -p /nsm/repo mkdir -p /opt/so/conf/reposync/cache - echo "https://repo.securityonion.net/file/so-repo/prod/2.4/oracle/9" > /opt/so/conf/reposync/mirror.txt - echo "https://so-repo-east.s3.us-east-005.backblazeb2.com/prod/2.4/oracle/9" >> /opt/so/conf/reposync/mirror.txt + echo "https://repo.securityonion.net/file/so-repo/prod/2.4/oracle/9.3" > /opt/so/conf/reposync/mirror.txt + echo "https://repo-alt.securityonion.net/prod/2.4/oracle/9.3" >> /opt/so/conf/reposync/mirror.txt echo "[main]" > /opt/so/conf/reposync/repodownload.conf echo "gpgcheck=1" >> /opt/so/conf/reposync/repodownload.conf echo "installonly_limit=3" >> /opt/so/conf/reposync/repodownload.conf From 25570e6ec2e9935c4a4481fd5f0ad32b9b7068de Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 21 Feb 2024 13:18:39 -0500 Subject: [PATCH 045/140] add missing template --- .github/DISCUSSION_TEMPLATE/2-4.yml | 190 ++++++++++++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 .github/DISCUSSION_TEMPLATE/2-4.yml diff --git a/.github/DISCUSSION_TEMPLATE/2-4.yml b/.github/DISCUSSION_TEMPLATE/2-4.yml new file mode 100644 index 0000000000..8e25920717 --- /dev/null +++ b/.github/DISCUSSION_TEMPLATE/2-4.yml @@ -0,0 +1,190 @@ +body: + - type: markdown + attributes: + value: | + ⚠️ This category is solely for conversations related to Security Onion 2.4 ⚠️ + + If your organization needs more immediate, enterprise grade professional support, with one-on-one virtual meetings and screensharing, contact us via our website: https://securityonion.com/support + - type: dropdown + attributes: + label: Version + description: Which version of Security Onion 2.4.x are you asking about? + options: + - + - 2.4 Pre-release (Beta, Release Candidate) + - 2.4.10 + - 2.4.20 + - 2.4.30 + - 2.4.40 + - 2.4.50 + - 2.4.60 + - 2.4.70 + - 2.4.80 + - 2.4.90 + - 2.4.100 + - Other (please provide detail below) + validations: + required: true + - type: dropdown + attributes: + label: Installation Method + description: How did you install Security Onion? + options: + - + - Security Onion ISO image + - Network installation on Red Hat derivative like Oracle, Rocky, Alma, etc. + - Network installation on Ubuntu + - Network installation on Debian + - Other (please provide detail below) + validations: + required: true + - type: dropdown + attributes: + label: Description + description: > + Is this discussion about installation, configuration, upgrading, or other? + options: + - + - installation + - configuration + - upgrading + - other (please provide detail below) + validations: + required: true + - type: dropdown + attributes: + label: Installation Type + description: > + When you installed, did you choose Import, Eval, Standalone, Distributed, or something else? + options: + - + - Import + - Eval + - Standalone + - Distributed + - other (please provide detail below) + validations: + required: true + - type: dropdown + attributes: + label: Location + description: > + Is this deployment in the cloud, on-prem with Internet access, or airgap? + options: + - + - cloud + - on-prem with Internet access + - airgap + - other (please provide detail below) + validations: + required: true + - type: dropdown + attributes: + label: Hardware Specs + description: > + Does your hardware meet or exceed the minimum requirements for your installation type as shown at https://docs.securityonion.net/en/2.4/hardware.html? + options: + - + - Meets minimum requirements + - Exceeds minimum requirements + - Does not meet minimum requirements + - other (please provide detail below) + validations: + required: true + - type: input + attributes: + label: CPU + description: How many CPU cores do you have? + validations: + required: true + - type: input + attributes: + label: RAM + description: How much RAM do you have? + validations: + required: true + - type: input + attributes: + label: Storage for / + description: How much storage do you have for the / partition? + validations: + required: true + - type: input + attributes: + label: Storage for /nsm + description: How much storage do you have for the /nsm partition? + validations: + required: true + - type: dropdown + attributes: + label: Network Traffic Collection + description: > + Are you collecting network traffic from a tap or span port? + options: + - + - tap + - span port + - other (please provide detail below) + validations: + required: true + - type: dropdown + attributes: + label: Network Traffic Speeds + description: > + How much network traffic are you monitoring? + options: + - + - Less than 1Gbps + - 1Gbps to 10Gbps + - more than 10Gbps + validations: + required: true + - type: dropdown + attributes: + label: Status + description: > + Does SOC Grid show all services on all nodes as running OK? + options: + - + - Yes, all services on all nodes are running OK + - No, one or more services are failed (please provide detail below) + validations: + required: true + - type: dropdown + attributes: + label: Salt Status + description: > + Do you get any failures when you run "sudo salt-call state.highstate"? + options: + - + - Yes, there are salt failures (please provide detail below) + - No, there are no failures + validations: + required: true + - type: dropdown + attributes: + label: Logs + description: > + Are there any additional clues in /opt/so/log/? + options: + - + - Yes, there are additional clues in /opt/so/log/ (please provide detail below) + - No, there are no additional clues + validations: + required: true + - type: textarea + attributes: + label: Detail + description: Please read our discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 and then provide detailed information to help us help you. + placeholder: |- + STOP! Before typing, please read our discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 in their entirety! + + If your organization needs more immediate, enterprise grade professional support, with one-on-one virtual meetings and screensharing, contact us via our website: https://securityonion.com/support + validations: + required: true + - type: checkboxes + attributes: + label: Guidelines + options: + - label: I have read the discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 and assert that I have followed the guidelines. + required: true From 162785575cbae9d9454a1d58830ab740129f1c57 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Feb 2024 15:28:24 -0500 Subject: [PATCH 046/140] nest under policy --- salt/elasticsearch/soc_elasticsearch.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 9a64190b35..e68d0441b5 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -95,6 +95,7 @@ elasticsearch: description: The order to sort by. Must set index_sorting to True. global: True helpLink: elasticsearch.html + policy: phases: hot: max_age: From 927ea0c9ecf01e4f8b08b6e2688abbefac8cf1d2 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 21 Feb 2024 15:56:12 -0500 Subject: [PATCH 047/140] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7f2e976173..5a99ed019d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.0-kilo +2.4.60 From 0a9022ba6a1f6ce82efa28d632b2f4f4bb1cd7b6 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 21 Feb 2024 17:07:08 -0500 Subject: [PATCH 048/140] Add hash mappings --- salt/soc/files/soc/sigma_so_pipeline.yaml | 32 ++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml index a1c4d6d62f..54ce83eff6 100644 --- a/salt/soc/files/soc/sigma_so_pipeline.yaml +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -15,4 +15,34 @@ transformations: src_ip: destination.ip.keyword src_port: source.port dst_ip: destination.ip.keyword - dst_port: destination.port \ No newline at end of file + dst_port: destination.port + - id: hashes_process-creation + type: field_name_mapping + mapping: + winlog.event_data.sha256: process.hash.sha256 + winlog.event_data.sha1: process.hash.sha1 + winlog.event_data.md5: process.hash.md5 + rule_conditions: + - type: logsource + product: windows + category: process_creation + - id: hashes_image-load + type: field_name_mapping + mapping: + winlog.event_data.sha256: dll.hash.sha256 + winlog.event_data.sha1: dll.hash.sha1 + winlog.event_data.md5: dll.hash.md5 + rule_conditions: + - type: logsource + product: windows + category: image_load + - id: hashes_driver-load + type: field_name_mapping + mapping: + winlog.event_data.sha256: dll.hash.sha256 + winlog.event_data.sha1: dll.hash.sha1 + winlog.event_data.md5: dll.hash.md5 + rule_conditions: + - type: logsource + product: windows + category: driver_load \ No newline at end of file From c886e7279363a8f0c614dc8a753166f5d418d5dc Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 22 Feb 2024 08:59:33 -0500 Subject: [PATCH 049/140] Imphash mappings --- salt/soc/files/soc/sigma_so_pipeline.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml index 54ce83eff6..8121a6f13f 100644 --- a/salt/soc/files/soc/sigma_so_pipeline.yaml +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -15,13 +15,15 @@ transformations: src_ip: destination.ip.keyword src_port: source.port dst_ip: destination.ip.keyword - dst_port: destination.port + dst_port: destination.port + winlog.event_data.User: user.name - id: hashes_process-creation type: field_name_mapping mapping: winlog.event_data.sha256: process.hash.sha256 winlog.event_data.sha1: process.hash.sha1 winlog.event_data.md5: process.hash.md5 + winlog.event_data.Imphash: process.pe.imphash rule_conditions: - type: logsource product: windows @@ -32,6 +34,7 @@ transformations: winlog.event_data.sha256: dll.hash.sha256 winlog.event_data.sha1: dll.hash.sha1 winlog.event_data.md5: dll.hash.md5 + winlog.event_data.Imphash: dll.pe.imphash rule_conditions: - type: logsource product: windows @@ -42,6 +45,7 @@ transformations: winlog.event_data.sha256: dll.hash.sha256 winlog.event_data.sha1: dll.hash.sha1 winlog.event_data.md5: dll.hash.md5 + winlog.event_data.Imphash: dll.pe.imphash rule_conditions: - type: logsource product: windows From 759b2ff59e5ffa201464a32a0ac776925a8044bb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 22 Feb 2024 10:03:51 -0500 Subject: [PATCH 050/140] Manage the repos --- salt/manager/files/mirror.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/manager/files/mirror.txt b/salt/manager/files/mirror.txt index e69de29bb2..732c116b41 100644 --- a/salt/manager/files/mirror.txt +++ b/salt/manager/files/mirror.txt @@ -0,0 +1,2 @@ +https://repo.securityonion.net/file/so-repo/prod/2.4/oracle/9 +https://repo-alt.securityonion.net/prod/2.4/oracle/9 \ No newline at end of file From e7914fc5a16bc47d243835244985088e717d9a80 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 22 Feb 2024 12:49:06 -0500 Subject: [PATCH 051/140] Update stenoloss.sh --- salt/telegraf/scripts/stenoloss.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/telegraf/scripts/stenoloss.sh b/salt/telegraf/scripts/stenoloss.sh index 5c27ee7a5d..5219dcfd0c 100644 --- a/salt/telegraf/scripts/stenoloss.sh +++ b/salt/telegraf/scripts/stenoloss.sh @@ -10,8 +10,8 @@ # if this script isn't already running if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then - CHECKIT=$(grep "Thread 0" /var/log/stenographer/stenographer.log |tac |head -2|wc -l) - STENOGREP=$(grep "Thread 0" /var/log/stenographer/stenographer.log |tac |head -2) + CHECKIT=$(grep "Thread 0 stats" /var/log/stenographer/stenographer.log |tac |head -2|wc -l) + STENOGREP=$(grep "Thread 0 stats" /var/log/stenographer/stenographer.log |tac |head -2) declare RESULT=($STENOGREP) From d04aa06455e7e96b5cbdac6c09953e01bd718c81 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 22 Feb 2024 14:01:02 -0500 Subject: [PATCH 052/140] Fix source.ip --- salt/soc/files/soc/sigma_so_pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml index 8121a6f13f..533823e6f2 100644 --- a/salt/soc/files/soc/sigma_so_pipeline.yaml +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -12,7 +12,7 @@ transformations: sid: rule.uuid answer: answers query: dns.query.name - src_ip: destination.ip.keyword + src_ip: source.ip.keyword src_port: source.port dst_ip: destination.ip.keyword dst_port: destination.port From b8baca417bc6a50a5149b1f538dba6e9eb26ffb4 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 23 Feb 2024 14:03:04 -0500 Subject: [PATCH 053/140] add endpoint_x_events_x_process to defaults.yaml --- salt/soc/defaults.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 2c15fe996f..d672d1dadb 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -995,6 +995,14 @@ soc: - tds.header_type - log.id.uid - event.dataset + ':endpoint:endpoint_x_events_x_process': + - soc_timestamp + - event.dataset + - host.name + - user.name + - process.parent.name + - process.name + - process.working_directory server: bindAddress: 0.0.0.0:9822 baseUrl: / From 573d565976b267b304ce6c03b5ca8a9041b4ad70 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 23 Feb 2024 15:03:44 -0500 Subject: [PATCH 054/140] convert _x_ to . for soc ui to config --- salt/soc/merged.map.jinja | 2 ++ salt/soc/soc_soc.yaml | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index 33c0070ade..55d8d26005 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -66,6 +66,8 @@ {% do SOCMERGED.config.server.client.alerts.update({'actions': standard_actions}) %} {% do SOCMERGED.config.server.client.cases.update({'actions': standard_actions}) %} +{# replace the _x_ with . for soc ui to config conversion #} +{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.process': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_process') }) %} {% set standard_eventFields = SOCMERGED.config.pop('eventFields') %} {% do SOCMERGED.config.server.client.hunt.update({'eventFields': standard_eventFields}) %} {% do SOCMERGED.config.server.client.dashboards.update({'eventFields': standard_eventFields}) %} diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index fdfb09733a..13e50a0cbe 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -55,10 +55,11 @@ soc: global: True forcedType: "[]{}" eventFields: - default: - description: Event fields mappings are defined by the format ":event.module:event.dataset". For example, to customize which fields show for 'syslog' events originating from 'zeek', find the eventField item in the left panel that looks like ':zeek:syslog'. This 'default' entry is used for all events that do not match an existing mapping defined in the list to the left. + default: &eventFields + description: Event fields mappings are defined by the format ":event.module:event.dataset". For example, to customize which fields show for 'syslog' events originating from 'zeek', find the eventField item in the left panel that looks like ':zeek:syslog'. The 'default' entry is used for all events that do not match an existing mapping defined in the list to the left. global: True advanced: True + ':endpoint:endpoint_x_events_x_process': *eventFields server: srvKey: description: Unique key for protecting the integrity of user submitted data via the web browser. From 7da0ccf5a628d71e75eef7fda547004ca693fec4 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 23 Feb 2024 15:35:53 -0500 Subject: [PATCH 055/140] add more endpoint.events.x entries to merged.map.jinja --- salt/soc/merged.map.jinja | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index 55d8d26005..c0ea836e52 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -67,7 +67,13 @@ {% do SOCMERGED.config.server.client.cases.update({'actions': standard_actions}) %} {# replace the _x_ with . for soc ui to config conversion #} +{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.api': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_api') }) %} +{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.file': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_file') }) %} +{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.library': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_library') }) %} +{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.network': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_network') }) %} {% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.process': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_process') }) %} +{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.registry': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_registry') }) %} +{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.security': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_security') }) %} {% set standard_eventFields = SOCMERGED.config.pop('eventFields') %} {% do SOCMERGED.config.server.client.hunt.update({'eventFields': standard_eventFields}) %} {% do SOCMERGED.config.server.client.dashboards.update({'eventFields': standard_eventFields}) %} From b7ef1e8af121862174fd72b0a75125fbf1341ec1 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 23 Feb 2024 15:38:53 -0500 Subject: [PATCH 056/140] add more endpoint.events.x fields to soc_soc.yaml --- salt/soc/soc_soc.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 13e50a0cbe..f59d6117b5 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -59,7 +59,13 @@ soc: description: Event fields mappings are defined by the format ":event.module:event.dataset". For example, to customize which fields show for 'syslog' events originating from 'zeek', find the eventField item in the left panel that looks like ':zeek:syslog'. The 'default' entry is used for all events that do not match an existing mapping defined in the list to the left. global: True advanced: True + ':endpoint:endpoint_x_events_x_api': *eventFields + ':endpoint:endpoint_x_events_x_file': *eventFields + ':endpoint:endpoint_x_events_x_library': *eventFields + ':endpoint:endpoint_x_events_x_network': *eventFields ':endpoint:endpoint_x_events_x_process': *eventFields + ':endpoint:endpoint_x_events_x_registry': *eventFields + ':endpoint:endpoint_x_events_x_security': *eventFields server: srvKey: description: Unique key for protecting the integrity of user submitted data via the web browser. From 58f4fb87d08813ab41eea5b24e6b2a4a2326d5ff Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 23 Feb 2024 17:06:29 -0500 Subject: [PATCH 057/140] fix new eventFields in soc_soc.yaml --- salt/soc/soc_soc.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index f59d6117b5..a9d36c70c7 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -59,13 +59,13 @@ soc: description: Event fields mappings are defined by the format ":event.module:event.dataset". For example, to customize which fields show for 'syslog' events originating from 'zeek', find the eventField item in the left panel that looks like ':zeek:syslog'. The 'default' entry is used for all events that do not match an existing mapping defined in the list to the left. global: True advanced: True - ':endpoint:endpoint_x_events_x_api': *eventFields - ':endpoint:endpoint_x_events_x_file': *eventFields - ':endpoint:endpoint_x_events_x_library': *eventFields - ':endpoint:endpoint_x_events_x_network': *eventFields - ':endpoint:endpoint_x_events_x_process': *eventFields - ':endpoint:endpoint_x_events_x_registry': *eventFields - ':endpoint:endpoint_x_events_x_security': *eventFields + ':endpoint:events_x_api': *eventFields + ':endpoint:events_x_file': *eventFields + ':endpoint:events_x_library': *eventFields + ':endpoint:events_x_network': *eventFields + ':endpoint:events_x_process': *eventFields + ':endpoint:events_x_registry': *eventFields + ':endpoint:events_x_security': *eventFields server: srvKey: description: Unique key for protecting the integrity of user submitted data via the web browser. From daf96d79342959155d2fccb7313c8e5a9b393f60 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 23 Feb 2024 17:07:48 -0500 Subject: [PATCH 058/140] fix new eventFields in merged.map.jinja --- salt/soc/merged.map.jinja | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index c0ea836e52..65091158e2 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -67,13 +67,13 @@ {% do SOCMERGED.config.server.client.cases.update({'actions': standard_actions}) %} {# replace the _x_ with . for soc ui to config conversion #} -{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.api': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_api') }) %} -{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.file': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_file') }) %} -{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.library': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_library') }) %} -{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.network': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_network') }) %} -{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.process': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_process') }) %} -{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.registry': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_registry') }) %} -{% do SOCMERGED.config.eventFields.update({':endpoint:endpoint.events.security': SOCMERGED.config.eventFields.pop(':endpoint:endpoint_x_events_x_security') }) %} +{% do SOCMERGED.config.eventFields.update({':endpoint:events.api': SOCMERGED.config.eventFields.pop(':endpoint:events_x_api') }) %} +{% do SOCMERGED.config.eventFields.update({':endpoint:events.file': SOCMERGED.config.eventFields.pop(':endpoint:events_x_file') }) %} +{% do SOCMERGED.config.eventFields.update({':endpoint:events.library': SOCMERGED.config.eventFields.pop(':endpoint:events_x_library') }) %} +{% do SOCMERGED.config.eventFields.update({':endpoint:events.network': SOCMERGED.config.eventFields.pop(':endpoint:events_x_network') }) %} +{% do SOCMERGED.config.eventFields.update({':endpoint:events.process': SOCMERGED.config.eventFields.pop(':endpoint:events_x_process') }) %} +{% do SOCMERGED.config.eventFields.update({':endpoint:events.registry': SOCMERGED.config.eventFields.pop(':endpoint:events_x_registry') }) %} +{% do SOCMERGED.config.eventFields.update({':endpoint:events.security': SOCMERGED.config.eventFields.pop(':endpoint:events_x_security') }) %} {% set standard_eventFields = SOCMERGED.config.pop('eventFields') %} {% do SOCMERGED.config.server.client.hunt.update({'eventFields': standard_eventFields}) %} {% do SOCMERGED.config.server.client.dashboards.update({'eventFields': standard_eventFields}) %} From d6cb8ab92823b95d0d35cee157549b52a91d2c97 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 23 Feb 2024 17:09:40 -0500 Subject: [PATCH 059/140] update events_x_process in defaults.yaml --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index d672d1dadb..7204027fc5 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -995,7 +995,7 @@ soc: - tds.header_type - log.id.uid - event.dataset - ':endpoint:endpoint_x_events_x_process': + ':endpoint:events_x_process': - soc_timestamp - event.dataset - host.name From a6bb7216f9aaead1b51887cee531e8b74e6272c2 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 26 Feb 2024 08:18:42 -0500 Subject: [PATCH 060/140] Add Detection AutoUpdate config --- salt/soc/defaults.yaml | 2 ++ salt/soc/soc_soc.yaml | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 2c15fe996f..5267955b9d 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1009,6 +1009,7 @@ soc: kratos: hostUrl: elastalertengine: + autoUpdateEnabled: false communityRulesImportFrequencySeconds: 180 elastAlertRulesFolder: /opt/sensoroni/elastalert rulesFingerprintFile: /opt/sensoroni/fingerprints/sigma.fingerprint @@ -1057,6 +1058,7 @@ soc: userFiles: - rbac/users_roles strelkaengine: + autoUpdateEnabled: false compileYaraPythonScriptPath: /opt/so/conf/strelka/compile_yara.py reposFolder: /opt/sensoroni/yara/repos rulesRepos: diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index fdfb09733a..74ae1051b8 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -75,6 +75,10 @@ soc: description: 'Defines the Sigma Community Ruleset you want to run. One of these (core | core+ | core++ | all ) as well as an optional Add-on (emerging_threats_addon). WARNING! Changing the ruleset will remove all existing Sigma rules of the previous ruleset and their associated overrides. This removal cannot be undone.' global: True advanced: False + autoUpdateEnabled: + description: 'Set to true to enable automatic updates of the Sigma Community Ruleset.' + global: True + advanced: True elastic: index: description: Comma-separated list of indices or index patterns (wildcard "*" supported) that SOC will search for records. @@ -133,6 +137,11 @@ soc: description: Duration (in milliseconds) to wait for a response from the Salt API when executing common grid management tasks before giving up and showing an error on the SOC UI. global: True advanced: True + strelkaengine: + autoUpdateEnabled: + description: 'Set to true to enable automatic updates of the Yara ruleset.' + global: True + advanced: True client: enableReverseLookup: description: Set to true to enable reverse DNS lookups for IP addresses in the SOC UI. From ca249312baddc9c3d6b7321f104ccc58da447acb Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Mon, 26 Feb 2024 09:38:14 -0500 Subject: [PATCH 061/140] FEATURE: Add new SOC action for Process Info #12421 --- salt/soc/defaults.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 7204027fc5..20f9c284f3 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -65,12 +65,18 @@ soc: target: _blank links: - 'https://{:sublime.url}/messages/{:sublime.message_group_id}' + - name: actionProcessInfo + description: actionProcessInfoHelp + icon: fa-person-running + target: '' + links: + - '/#/hunt?q=(process.entity_id:"{:process.entity_id}") | groupby event.dataset | groupby -sankey event.dataset event.action | groupby event.action | groupby process.name | groupby host.name user.name | groupby source.ip source.port destination.ip destination.port | groupby dns.question.name | groupby dns.answers.data | groupby file.path | groupby registry.path | groupby dll.path' - name: actionProcessAncestors description: actionProcessAncestorsHelp icon: fa-people-roof target: '' links: - - '/#/hunt?q=(process.entity_id:"{:process.entity_id}" OR process.entity_id:"{:process.Ext.ancestry|processAncestors}") | groupby process.parent.name | groupby -sankey process.parent.name process.name | groupby process.name | groupby event.module event.dataset | table soc_timestamp event.dataset host.name user.name process.parent.name process.name process.working_directory' + - '/#/hunt?q=(process.entity_id:"{:process.entity_id}" OR process.entity_id:"{:process.Ext.ancestry|processAncestors}") | groupby event.dataset | groupby -sankey event.dataset event.action | groupby event.action | groupby process.parent.name | groupby -sankey process.parent.name process.name | groupby process.name | groupby host.name user.name | groupby source.ip source.port destination.ip destination.port | groupby dns.question.name | groupby dns.answers.data | groupby file.path | groupby registry.path | groupby dll.path' eventFields: default: - soc_timestamp From 4df21148fc4b7117b6a69b2914d961855d3f12e7 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Mon, 26 Feb 2024 09:40:51 -0500 Subject: [PATCH 062/140] FEATURE: Add default columns for endpoint.events datasets #12425 --- salt/soc/defaults.yaml | 57 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 20f9c284f3..b5fe0e6262 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1001,14 +1001,69 @@ soc: - tds.header_type - log.id.uid - event.dataset - ':endpoint:events_x_process': + ':endpoint:events_x_api': + - soc_timestamp + - host.name + - user.name + - process.name + - process.Ext.api.name + - process.thread.Ext.call_stack_final_user_module.path + - event.dataset + ':endpoint:events_x_file': + - soc_timestamp + - host.name + - user.name + - process.name + - event.action + - file.path + - event.dataset + ':endpoint:events_x_library': - soc_timestamp + - host.name + - user.name + - process.name + - event.action + - dll.path + - dll.code_signature.status + - dll.code_signature.subject_name - event.dataset + ':endpoint:events_x_network': + - soc_timestamp + - host.name + - user.name + - process.name + - event.action + - source.ip + - source.port + - destination.ip + - destination.port + - network.community_id + - event.dataset + ':endpoint:events_x_process': + - soc_timestamp - host.name - user.name - process.parent.name - process.name + - event.action - process.working_directory + - event.dataset + ':endpoint:events_x_registry': + - soc_timestamp + - host.name + - user.name + - process.name + - event.action + - registry.path + - event.dataset + ':endpoint:events_x_security': + - soc_timestamp + - host.name + - user.name + - process.executable + - event.action + - event.outcome + - event.dataset server: bindAddress: 0.0.0.0:9822 baseUrl: / From c8a95a87069a996543672c50485280394e45226d Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Mon, 26 Feb 2024 09:59:07 -0500 Subject: [PATCH 063/140] FEATURE: Add new endpoint dashboards #12428 --- salt/soc/defaults.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index b5fe0e6262..c1b3ebabbd 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1502,13 +1502,22 @@ soc: query: 'event.category: network AND _exists_:process.executable AND (_exists_:dns.question.name OR _exists_:dns.answers.data) | groupby -sankey host.name dns.question.name | groupby event.dataset event.type | groupby host.name | groupby process.executable | groupby dns.question.name | groupby dns.answers.data' - name: Host Process Activity description: Process activity captured on an endpoint - query: 'event.category:process | groupby -sankey host.name user.name* | groupby event.dataset event.action | groupby host.name | groupby user.name | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable | table soc_timestamp event.dataset host.name user.name process.parent.name process.name process.working_directory' + query: 'event.category:process | groupby -sankey host.name user.name* | groupby event.dataset event.action | groupby host.name | groupby user.name | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable' - name: Host File Activity description: File activity captured on an endpoint query: 'event.category: file AND _exists_:process.executable | groupby -sankey host.name process.executable | groupby host.name | groupby event.dataset event.action event.type | groupby file.name | groupby process.executable' - name: Host Network & Process Mappings description: Network activity mapped to originating processes query: 'event.category: network AND _exists_:process.executable | groupby -sankey event.action host.name | groupby -sankey host.name user.name | groupby event.dataset* event.type* event.action* | groupby host.name | groupby user.name | groupby dns.question.name | groupby process.executable | groupby winlog.event_data.TargetObject | groupby process.name | groupby source.ip | groupby destination.ip | groupby destination.port' + - name: Host API Events + description: API (Application Programming Interface) events from endpoints + query: 'event.dataset:endpoint.events.api | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby process.name | groupby process.Ext.api.name' + - name: Host Library Events + description: Library events from endpoints + query: 'event.dataset:endpoint.events.library | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby process.name | groupby event.action | groupby dll.path | groupby dll.code_signature.status | groupby dll.code_signature.subject_name' + - name: Host Security Events + description: Security events from endpoints + query: 'event.dataset:endpoint.events.security | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby process.executable | groupby event.action | groupby event.outcome' - name: Strelka description: Strelka file analysis query: 'event.module:strelka | groupby file.mime_type | groupby -sankey file.mime_type file.source | groupby file.source | groupby file.name' From 9a7e2153eedec1fbeb61df3db918ba5b7e7baa39 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 26 Feb 2024 11:01:53 -0500 Subject: [PATCH 064/140] add classification.config --- salt/suricata/classification/classification.config | 2 ++ salt/suricata/config.sls | 7 +++++++ salt/suricata/enabled.sls | 1 + salt/suricata/soc_suricata.yaml | 7 +++++++ 4 files changed, 17 insertions(+) create mode 100644 salt/suricata/classification/classification.config diff --git a/salt/suricata/classification/classification.config b/salt/suricata/classification/classification.config new file mode 100644 index 0000000000..69918fed71 --- /dev/null +++ b/salt/suricata/classification/classification.config @@ -0,0 +1,2 @@ +# configuration classification: shortname,description,priority +# configuration classification: misc-activity,Misc activity,3 diff --git a/salt/suricata/config.sls b/salt/suricata/config.sls index 3ec1324bf0..00364f3840 100644 --- a/salt/suricata/config.sls +++ b/salt/suricata/config.sls @@ -129,6 +129,13 @@ surithresholding: - group: 940 - template: jinja +suriclassifications: + file.managed: + - name: /opt/so/conf/suricata/classification.config + - source: salt://suricata/classification/classification.config + - user: 940 + - group: 940 + # BPF compilation and configuration {% if SURICATABPF %} {% set BPF_CALC = salt['cmd.script']('salt://common/tools/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + SURICATABPF|join(" "),cwd='/root') %} diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index ce309e41a4..f96472ae2d 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -27,6 +27,7 @@ so-suricata: - binds: - /opt/so/conf/suricata/suricata.yaml:/etc/suricata/suricata.yaml:ro - /opt/so/conf/suricata/threshold.conf:/etc/suricata/threshold.conf:ro + - /opt/so/conf/suricata/classification.config:/etc/suricata/classification.config:ro - /opt/so/conf/suricata/rules:/etc/suricata/rules:ro - /opt/so/log/suricata/:/var/log/suricata/:rw - /nsm/suricata/:/nsm/:rw diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index 30f277c0a7..4fd720ef14 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -11,6 +11,13 @@ suricata: multiline: True title: SIDS helpLink: suricata.html + classification: + classification__config: + description: Classifications config file. + file: True + global: True + multiline: True + helpLink: suricata.html config: af-packet: interface: From f8424f3dad29bed27f0d02006cb3af14f5f76e39 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Mon, 26 Feb 2024 11:22:09 -0500 Subject: [PATCH 065/140] Update defaults.yaml --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index c1b3ebabbd..d86262fe76 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1502,7 +1502,7 @@ soc: query: 'event.category: network AND _exists_:process.executable AND (_exists_:dns.question.name OR _exists_:dns.answers.data) | groupby -sankey host.name dns.question.name | groupby event.dataset event.type | groupby host.name | groupby process.executable | groupby dns.question.name | groupby dns.answers.data' - name: Host Process Activity description: Process activity captured on an endpoint - query: 'event.category:process | groupby -sankey host.name user.name* | groupby event.dataset event.action | groupby host.name | groupby user.name | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable' + query: 'event.category:process | groupby -sankey host.name user.name* | groupby event.dataset event.action | groupby host.name | groupby user.name | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable | table soc_timestamp host.name user.name process.parent.name process.name event.action process.working_directory event.dataset' - name: Host File Activity description: File activity captured on an endpoint query: 'event.category: file AND _exists_:process.executable | groupby -sankey host.name process.executable | groupby host.name | groupby event.dataset event.action event.type | groupby file.name | groupby process.executable' From 1d099f97d2577077607152ae2579e974936db53a Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 26 Feb 2024 11:27:56 -0500 Subject: [PATCH 066/140] Update pattern for endpoint diagnostic template --- salt/elasticsearch/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index f4d8c8a95e..8e28f2e41c 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -3926,7 +3926,7 @@ elasticsearch: allow_custom_routing: false hidden: false index_patterns: - - logs-endpoint.diagnostic.collection-* + - .logs-endpoint.diagnostic.collection-* priority: 501 template: settings: From 466dac30bbf0f7b6d3d2d065ea8eed3639541ac8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 26 Feb 2024 12:15:17 -0500 Subject: [PATCH 067/140] soup for classifications --- salt/manager/tools/sbin/soup | 15 +++++++++++++++ salt/suricata/soc_suricata.yaml | 1 + 2 files changed, 16 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 90ec636eff..752ae6e213 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -356,6 +356,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.4.20 ]] && up_to_2.4.30 [[ "$INSTALLEDVERSION" == 2.4.30 ]] && up_to_2.4.40 [[ "$INSTALLEDVERSION" == 2.4.40 ]] && up_to_2.4.50 + [[ "$INSTALLEDVERSION" == 2.4.50 ]] && up_to_2.4.60 true } @@ -371,6 +372,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == 2.4.20 ]] && post_to_2.4.30 [[ "$POSTVERSION" == 2.4.30 ]] && post_to_2.4.40 [[ "$POSTVERSION" == 2.4.40 ]] && post_to_2.4.50 + [[ "$POSTVERSION" == 2.4.50 ]] && post_to_2.4.60 true } @@ -427,6 +429,11 @@ post_to_2.4.50() { POSTVERSION=2.4.50 } +post_to_2.4.60() { + echo "Nothing to apply" + POSTVERSION=2.4.60 +} + repo_sync() { echo "Sync the local repo." su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync." @@ -556,6 +563,14 @@ up_to_2.4.50() { INSTALLEDVERSION=2.4.50 } +up_to_2.4.60() { + echo "Creating directory to store Suricata classification.config" + mkdir -vp /opt/so/saltstack/local/salt/suricata/classification + chown socore:socore /opt/so/saltstack/local/salt/suricata/classification + + INSTALLEDVERSION=2.4.60 +} + determine_elastic_agent_upgrade() { if [[ $is_airgap -eq 0 ]]; then update_elastic_agent_airgap diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index 4fd720ef14..b54a44cbc9 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -17,6 +17,7 @@ suricata: file: True global: True multiline: True + title: Classifications helpLink: suricata.html config: af-packet: From 8b7f7933bdfa4e67aa7e3a84dd1c3d03340935c7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 26 Feb 2024 15:29:13 -0500 Subject: [PATCH 068/140] suricata container watch classification.config --- salt/suricata/classification/classification.config | 4 ++-- salt/suricata/enabled.sls | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/suricata/classification/classification.config b/salt/suricata/classification/classification.config index 69918fed71..e597eb5a13 100644 --- a/salt/suricata/classification/classification.config +++ b/salt/suricata/classification/classification.config @@ -1,2 +1,2 @@ -# configuration classification: shortname,description,priority -# configuration classification: misc-activity,Misc activity,3 +# config classification: shortname,description,priority +# config classification: misc-activity,Misc activity,3 diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index f96472ae2d..94b95ff5d6 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -50,10 +50,12 @@ so-suricata: - file: surithresholding - file: /opt/so/conf/suricata/rules/ - file: /opt/so/conf/suricata/bpf + - file: suriclassifications - require: - file: suriconfig - file: surithresholding - file: suribpf + - file: suriclassifications delete_so-suricata_so-status.disabled: file.uncomment: From c6baa4be1baad88a7301e7c08d8c960d865afca0 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 26 Feb 2024 16:19:32 -0500 Subject: [PATCH 069/140] Airgap Support - Detections module --- salt/manager/tools/sbin/soup | 10 ++++++---- salt/soc/enabled.sls | 2 +- salt/soc/merged.map.jinja | 6 ++++++ setup/so-setup | 6 ++---- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 90ec636eff..655e99f6c2 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -603,6 +603,10 @@ update_airgap_rules() { if [ -d /nsm/repo/rules/sigma ]; then rsync -av $UPDATE_DIR/agrules/sigma/* /nsm/repo/rules/sigma/ fi + + # SOC Detections Airgap + rsync -av $UPDATE_DIR/agrules/detect-sigma/* /nsm/rules/detect-sigma/ + rsync -av $UPDATE_DIR/agrules/detect-yara/* /nsm/rules/detect-yara/ } update_airgap_repo() { @@ -931,10 +935,8 @@ main() { preupgrade_changes echo "" - if [[ $is_airgap -eq 0 ]]; then - echo "Updating Rule Files to the Latest." - update_airgap_rules - fi + echo "Updating Airgap Rule Files to the Latest." + update_airgap_rules # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars if [[ ! "$MINIONID" =~ "_import" ]]; then diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 7c04da8255..93ca07ac88 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -22,7 +22,7 @@ so-soc: - sobridge: - ipv4_address: {{ DOCKER.containers['so-soc'].ip }} - binds: - - /nsm/rules:/nsm/rules:rw #Need to tighten this up? + - /nsm/rules:/nsm/rules:rw - /opt/so/conf/strelka:/opt/sensoroni/yara:rw - /opt/so/rules/elastalert/rules:/opt/sensoroni/elastalert:rw - /opt/so/conf/soc/fingerprints:/opt/sensoroni/fingerprints:rw diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index 65091158e2..bc7c5cada7 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -34,6 +34,12 @@ {% do SOCMERGED.config.server.client.inactiveTools.append('toolPlaybook') %} {% endif %} +{# if system is Airgap, don't autoupdate Yara & Sigma rules #} +{% if pillar.global.airgap %} + {% do SOCMERGED.config.server.modules.elastalertengine.update({'autoUpdateEnabled': false}) %} + {% do SOCMERGED.config.server.modules.strelkaengine.update({'autoUpdateEnabled': false}) %} +{% endif %} + {% set standard_actions = SOCMERGED.config.pop('actions') %} {% if pillar.global.endgamehost != '' %} diff --git a/setup/so-setup b/setup/so-setup index ca1581ef90..e2de39f50e 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -687,10 +687,8 @@ if ! [[ -f $install_opt_file ]]; then logCmd "so-minion -o=setup" title "Creating Global SLS" - if [[ $is_airgap ]]; then - # Airgap Rules - airgap_rules - fi + # Airgap Rules + airgap_rules manager_pillar From 59af547838100d006f924ad43e302c7aac736bc6 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 27 Feb 2024 09:49:54 -0500 Subject: [PATCH 070/140] Fix download location --- salt/soc/soc_soc.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index fa488ab7a2..e80e98a7e5 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -83,7 +83,7 @@ soc: global: True advanced: False autoUpdateEnabled: - description: 'Set to true to enable automatic updates of the Sigma Community Ruleset.' + description: 'Set to true to enable automatic Internet-connected updates of the Sigma Community Ruleset. If this is an Airgap system, this setting will be overridden and set to false.' global: True advanced: True elastic: @@ -146,7 +146,7 @@ soc: advanced: True strelkaengine: autoUpdateEnabled: - description: 'Set to true to enable automatic updates of the Yara ruleset.' + description: 'Set to true to enable automatic Internet-connected updates of the Yara rulesets. If this is an Airgap system, this setting will be overridden and set to false.' global: True advanced: True client: From fcc0f9d14f50019dcad5ffc02035173fe50c6bbe Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 27 Feb 2024 13:20:58 -0500 Subject: [PATCH 071/140] redo classifications --- .../classification/classification.config | 53 ++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/salt/suricata/classification/classification.config b/salt/suricata/classification/classification.config index e597eb5a13..220736c947 100644 --- a/salt/suricata/classification/classification.config +++ b/salt/suricata/classification/classification.config @@ -1,2 +1,51 @@ -# config classification: shortname,description,priority -# config classification: misc-activity,Misc activity,3 +# +# config classification:shortname,short description,priority +# + +config classification: not-suspicious,Not Suspicious Traffic,3 +config classification: unknown,Unknown Traffic,3 +config classification: bad-unknown,Potentially Bad Traffic, 2 +config classification: attempted-recon,Attempted Information Leak,2 +config classification: successful-recon-limited,Information Leak,2 +config classification: successful-recon-largescale,Large Scale Information Leak,2 +config classification: attempted-dos,Attempted Denial of Service,2 +config classification: successful-dos,Denial of Service,2 +config classification: attempted-user,Attempted User Privilege Gain,1 +config classification: unsuccessful-user,Unsuccessful User Privilege Gain,1 +config classification: successful-user,Successful User Privilege Gain,1 +config classification: attempted-admin,Attempted Administrator Privilege Gain,1 +config classification: successful-admin,Successful Administrator Privilege Gain,1 + +# NEW CLASSIFICATIONS +config classification: rpc-portmap-decode,Decode of an RPC Query,2 +config classification: shellcode-detect,Executable code was detected,1 +config classification: string-detect,A suspicious string was detected,3 +config classification: suspicious-filename-detect,A suspicious filename was detected,2 +config classification: suspicious-login,An attempted login using a suspicious username was detected,2 +config classification: system-call-detect,A system call was detected,2 +config classification: tcp-connection,A TCP connection was detected,4 +config classification: trojan-activity,A Network Trojan was detected, 1 +config classification: unusual-client-port-connection,A client was using an unusual port,2 +config classification: network-scan,Detection of a Network Scan,3 +config classification: denial-of-service,Detection of a Denial of Service Attack,2 +config classification: non-standard-protocol,Detection of a non-standard protocol or event,2 +config classification: protocol-command-decode,Generic Protocol Command Decode,3 +config classification: web-application-activity,access to a potentially vulnerable web application,2 +config classification: web-application-attack,Web Application Attack,1 +config classification: misc-activity,Misc activity,3 +config classification: misc-attack,Misc Attack,2 +config classification: icmp-event,Generic ICMP event,3 +config classification: inappropriate-content,Inappropriate Content was Detected,1 +config classification: policy-violation,Potential Corporate Privacy Violation,1 +config classification: default-login-attempt,Attempt to login by a default username and password,2 + +# Update +config classification: targeted-activity,Targeted Malicious Activity was Detected,1 +config classification: exploit-kit,Exploit Kit Activity Detected,1 +config classification: external-ip-check,Device Retrieving External IP Address Detected,2 +config classification: domain-c2,Domain Observed Used for C2 Detected,1 +config classification: pup-activity,Possibly Unwanted Program Detected,2 +config classification: credential-theft,Successful Credential Theft Detected,1 +config classification: social-engineering,Possible Social Engineering Attempted,2 +config classification: coin-mining,Crypto Currency Mining Activity Detected,2 +config classification: command-and-control,Malware Command and Control Activity Detected,1 From df3943b4651e1bf293e0433d2138dece8ffc829b Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 27 Feb 2024 17:24:27 -0500 Subject: [PATCH 072/140] Daily rollover --- salt/elasticsearch/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 8e28f2e41c..39d2185643 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -10597,7 +10597,7 @@ elasticsearch: hot: actions: rollover: - max_age: 30d + max_age: 1d max_primary_shard_size: 50gb set_priority: priority: 100 From e2dd0f8cf17950a6b35d66419085adb167a28e34 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 28 Feb 2024 09:39:23 -0500 Subject: [PATCH 073/140] Only update rule files if AG --- salt/manager/tools/sbin/soup | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 655e99f6c2..ba83161163 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -935,8 +935,10 @@ main() { preupgrade_changes echo "" - echo "Updating Airgap Rule Files to the Latest." - update_airgap_rules + if [[ $is_airgap -eq 0 ]]; then + echo "Updating Rule Files to the Latest." + update_airgap_rules + fi # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars if [[ ! "$MINIONID" =~ "_import" ]]; then From 53761d4dba278243222822f1a00a1ec9ba033891 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 29 Feb 2024 16:15:26 -0500 Subject: [PATCH 074/140] FIX: EA installers not downloadable from SOC + fix stg logging Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/stig/enabled.sls | 21 ++++++++++++--------- salt/stig/files/sos-oscap.xml | 6 +++--- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/salt/stig/enabled.sls b/salt/stig/enabled.sls index 1f1a064fd8..0f347f8bcd 100644 --- a/salt/stig/enabled.sls +++ b/salt/stig/enabled.sls @@ -48,15 +48,17 @@ update_stig_profile: {% if not salt['file.file_exists'](OSCAP_OUTPUT_DIR ~ '/pre-oscap-report.html') %} run_initial_scan: - module.run: - - name: openscap.xccdf - - params: 'eval --profile {{ OSCAP_PROFILE_NAME }} --results {{ OSCAP_OUTPUT_DIR }}/pre-oscap-results.xml --report {{ OSCAP_OUTPUT_DIR }}/pre-oscap-report.html {{ OSCAP_PROFILE_LOCATION }}' + cmd.run: + - name: 'oscap xccdf eval --profile {{ OSCAP_PROFILE_NAME }} --results {{ OSCAP_OUTPUT_DIR }}/pre-oscap-results.xml --report {{ OSCAP_OUTPUT_DIR }}/pre-oscap-report.html {{ OSCAP_PROFILE_LOCATION }}' + - success_retcodes: + - 2 {% endif %} run_remediate: - module.run: - - name: openscap.xccdf - - params: 'eval --remediate --profile {{ OSCAP_PROFILE_NAME }} --results {{ OSCAP_OUTPUT_DIR }}/post-oscap-results.xml --report {{ OSCAP_PROFILE_LOCATION }}' + cmd.run: + - name: 'oscap xccdf eval --remediate --profile {{ OSCAP_PROFILE_NAME }} {{ OSCAP_PROFILE_LOCATION }}' + - success_retcodes: + - 2 {# OSCAP rule id: xccdf_org.ssgproject.content_rule_disable_ctrlaltdel_burstaction #} disable_ctrl_alt_del_action: @@ -82,9 +84,10 @@ remove_nullok_from_system_auth_auth: - backup: '.bak' run_post_scan: - module.run: - - name: openscap.xccdf - - params: 'eval --profile {{ OSCAP_PROFILE_NAME }} --results {{ OSCAP_OUTPUT_DIR }}/post-oscap-results.xml --report {{ OSCAP_OUTPUT_DIR }}/post-oscap-report.html {{ OSCAP_PROFILE_LOCATION }}' + cmd.run: + - name: 'oscap xccdf eval --profile {{ OSCAP_PROFILE_NAME }} --results {{ OSCAP_OUTPUT_DIR }}/post-oscap-results.xml --report {{ OSCAP_OUTPUT_DIR }}/post-oscap-report.html {{ OSCAP_PROFILE_LOCATION }}' + - success_retcodes: + - 2 {% else %} {{sls}}_no_license_detected: diff --git a/salt/stig/files/sos-oscap.xml b/salt/stig/files/sos-oscap.xml index 3f78af8c09..6c4c93778b 100644 --- a/salt/stig/files/sos-oscap.xml +++ b/salt/stig/files/sos-oscap.xml @@ -611,7 +611,7 @@ the release. Additionally, the original security profile has been modified by Se - + @@ -1007,8 +1007,8 @@ the release. Additionally, the original security profile has been modified by Se - - + + From b017157d21a7b32a8163b7593d2279a06d3a776f Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 1 Mar 2024 14:04:56 -0500 Subject: [PATCH 075/140] Add antivirus mapping --- salt/soc/files/soc/sigma_so_pipeline.yaml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml index 533823e6f2..37e9f4a3ef 100644 --- a/salt/soc/files/soc/sigma_so_pipeline.yaml +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -16,7 +16,25 @@ transformations: src_port: source.port dst_ip: destination.ip.keyword dst_port: destination.port - winlog.event_data.User: user.name + winlog.event_data.User: user.name + # Maps "antivirus" category to Windows Defender logs shipped by Elastic Agent Winlog Integration + # winlog.event_data.threat_name has to be renamed prior to ingestion, it is originally winlog.event_data.Threat Name + - id: antivirus_field-mappings_windows-defender + type: field_name_mapping + mapping: + Signature: winlog.event_data.threat_name + rule_conditions: + - type: logsource + category: antivirus + - id: antivirus_add-fields_windows-defender + type: add_condition + conditions: + winlog.channel: 'Microsoft-Windows-Windows Defender/Operational' + winlog.provider_name: 'Microsoft-Windows-Windows Defender' + event.code: "1116" + rule_conditions: + - type: logsource + category: antivirus - id: hashes_process-creation type: field_name_mapping mapping: From d832158cc52fe7c87d88fe233c38128d425d0a2f Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 1 Mar 2024 15:26:02 -0500 Subject: [PATCH 076/140] Drop Hashes field --- salt/soc/files/soc/sigma_so_pipeline.yaml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml index 37e9f4a3ef..d227c3f01e 100644 --- a/salt/soc/files/soc/sigma_so_pipeline.yaml +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -35,6 +35,17 @@ transformations: rule_conditions: - type: logsource category: antivirus + # Drops the Hashes field which is specific to Sysmon logs + # Ingested sysmon logs will have the Hashes field mapped to ECS specific fields + - id: hashes_drop_sysmon-specific-field + type: drop_detection_item + field_name_conditions: + - type: include_fields + fields: + - winlog.event_data.Hashes + rule_conditions: + - type: logsource + product: windows - id: hashes_process-creation type: field_name_mapping mapping: @@ -67,4 +78,4 @@ transformations: rule_conditions: - type: logsource product: windows - category: driver_load \ No newline at end of file + category: driver_load From f28f269bb120195d00066d7b7fcf0304906c8e86 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 4 Mar 2024 07:38:32 -0500 Subject: [PATCH 077/140] Fix FIM --- .../tools/sbin_jinja/so-elastic-agent-gen-installers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers index 275bc6a116..ff46a3e072 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers @@ -46,7 +46,7 @@ do done printf "\n### Stripping out unused components" -find /nsm/elastic-agent-workspace/elastic-agent-*/data/elastic-agent-*/components -maxdepth 1 -regex '.*fleet.*\|.*packet.*\|.*apm.*\|.*audit.*\|.*heart.*\|.*cloud.*' -delete +find /nsm/elastic-agent-workspace/elastic-agent-*/data/elastic-agent-*/components -maxdepth 1 -regex '.*fleet.*\|.*packet.*\|.*apm.*\|.*heart.*\|.*cloud.*' -delete printf "\n### Tarring everything up again" for OS in "${OSARCH[@]}" From 018e099111e86774e51126b42d22253b73ea7495 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 4 Mar 2024 14:53:15 -0500 Subject: [PATCH 078/140] Modify setup --- salt/manager/tools/sbin/so-minion | 39 +++++++++++++++++++------------ 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 4995e1c9df..4a6e5b7c72 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -80,28 +80,27 @@ function getinstallinfo() { } function pcapspace() { - - local NSMSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/nsm"."1K-blocks" ') - local ROOTSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/"."1K-blocks" ') - - if [[ "$NSMSIZE" == "null" ]]; then - # Looks like there is no dedicated nsm partition. Using root - local SPACESIZE=$ROOTSIZE + if [[ "$OPERATION" == "setup" ]]; then + local SPACESIZE=$(df -h /nsm | tail -1 | awk '{print $2}') else - local SPACESIZE=$NSMSIZE + + local NSMSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/nsm"."1K-blocks" ') + local ROOTSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/"."1K-blocks" ') + + if [[ "$NSMSIZE" == "null" ]]; then + # Looks like there is no dedicated nsm partition. Using root + local SPACESIZE=$ROOTSIZE + else + local SPACESIZE=$NSMSIZE + fi fi local s=$(( $SPACESIZE / 1000000 )) local s1=$(( $s / 2 )) local s2=$(( $s1 / $lb_procs )) - printf '%s\n'\ - "suricata:"\ - " config:"\ - " output:"\ - " pcap-log:"\ - " max-files: $s" >> $PILLARFILE - + MAXPCAPFILES=$s2 + } function testMinion() { @@ -272,6 +271,11 @@ function add_sensor_to_minion() { echo " config:" >> $PILLARFILE echo " af-packet:" >> $PILLARFILE echo " threads: '$CORECOUNT'" >> $PILLARFILE + if [[ $is_pcaplimit ]]; then + echo " output:" >> $PILLARFILE + echo " pcap-log:" >> $PILLARFILE + echo " max-files: '$MAXPCAPFILES'" >> $PILLARFILE + fi echo "pcap:" >> $PILLARFILE echo " enabled: True" >> $PILLARFILE if [[ $is_pcaplimit ]]; then @@ -448,6 +452,7 @@ function updateMine() { function createEVAL() { is_pcaplimit=true + pcapspace add_elasticsearch_to_minion add_sensor_to_minion add_strelka_to_minion @@ -468,6 +473,7 @@ function createEVAL() { function createSTANDALONE() { is_pcaplimit=true + pcapspace add_elasticsearch_to_minion add_logstash_to_minion add_sensor_to_minion @@ -557,6 +563,7 @@ function createIDH() { function createHEAVYNODE() { is_pcaplimit=true + pcapspace add_elasticsearch_to_minion add_elastic_agent_to_minion add_logstash_to_minion @@ -567,6 +574,8 @@ function createHEAVYNODE() { } function createSENSOR() { + is_pcaplimit=true + pcapspace add_sensor_to_minion add_strelka_to_minion add_telegraf_to_minion From fe238755e981fdc3316c48f03c23c9a6a29613ca Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 4 Mar 2024 16:52:51 -0500 Subject: [PATCH 079/140] Fix df --- salt/manager/tools/sbin/so-minion | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 4a6e5b7c72..d696e14c68 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -81,7 +81,7 @@ function getinstallinfo() { function pcapspace() { if [[ "$OPERATION" == "setup" ]]; then - local SPACESIZE=$(df -h /nsm | tail -1 | awk '{print $2}') + local SPACESIZE=$(df -k /nsm | tail -1 | awk '{print $2}' | tr -d \n) else local NSMSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/nsm"."1K-blocks" ') From b64d61065a7cf910a1c53d28485bfbee21edb4e9 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 5 Mar 2024 09:19:43 -0500 Subject: [PATCH 080/140] Add AWS Cloudfront template --- salt/elasticsearch/defaults.yaml | 44 ++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 39d2185643..0d2dd8a41e 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1107,6 +1107,50 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-aws_x_cloudfront_logs: + index_sorting: False + index_template: + index_patterns: + - "logs-aws.cloudfront_logs-*" + template: + settings: + index: + lifecycle: + name: so-logs-aws.cloudfront_logs-logs + number_of_replicas: 0 + composed_of: + - "logs-aws.cloudfront_logs@package" + - "logs-aws.cloudfront_logs@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-aws_x_cloudtrail: index_sorting: false index_template: From 1514f1291e2961dedd91354c0593ffa6e0854023 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 5 Mar 2024 09:21:48 -0500 Subject: [PATCH 081/140] Add AWS GuardDuty template --- salt/elasticsearch/defaults.yaml | 44 ++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 0d2dd8a41e..54a65a1126 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1371,6 +1371,50 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-aws_x_guardduty: + index_sorting: False + index_template: + index_patterns: + - "logs-aws.guardduty-*" + template: + settings: + index: + lifecycle: + name: so-logs-aws.guardduty-logs + number_of_replicas: 0 + composed_of: + - "logs-aws.guardduty@package" + - "logs-aws.guardduty@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-aws_x_route53_public_logs: index_sorting: false index_template: From d85ac39e2875dac3a58930abd2523f5be7af6ece Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 5 Mar 2024 09:23:17 -0500 Subject: [PATCH 082/140] Add AWS Inspector template --- salt/elasticsearch/defaults.yaml | 44 ++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 54a65a1126..8d31a1acdc 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1415,6 +1415,50 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-aws_x_inspector: + index_sorting: False + index_template: + index_patterns: + - "logs-aws.inspector-*" + template: + settings: + index: + lifecycle: + name: so-logs-aws.inspector-logs + number_of_replicas: 0 + composed_of: + - "logs-aws.inspector@package" + - "logs-aws.inspector@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-aws_x_route53_public_logs: index_sorting: false index_template: From d8e8933ea0a035e0628a0bbcc65096a0c0a00b01 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 5 Mar 2024 09:25:41 -0500 Subject: [PATCH 083/140] Add AWS Security Hub template --- salt/elasticsearch/defaults.yaml | 88 ++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 8d31a1acdc..2274018b16 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1591,6 +1591,94 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-aws_x_securityhub_findings: + index_sorting: False + index_template: + index_patterns: + - "logs-aws.securityhub_findings-*" + template: + settings: + index: + lifecycle: + name: so-logs-aws.securityhub_findings-logs + number_of_replicas: 0 + composed_of: + - "logs-aws.securityhub_findings@package" + - "logs-aws.securityhub_findings@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-aws_x_securityhub_insights: + index_sorting: False + index_template: + index_patterns: + - "logs-aws.securityhub_insights-*" + template: + settings: + index: + lifecycle: + name: so-logs-aws.securityhub_insights-logs + number_of_replicas: 0 + composed_of: + - "logs-aws.securityhub_insights@package" + - "logs-aws.securityhub_insights@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-aws_x_vpcflow: index_sorting: false index_template: From 2a7e5b096f0320dec1d395bf3fe3c5721582283f Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 5 Mar 2024 09:48:59 -0500 Subject: [PATCH 084/140] Change version for foxtrot --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 5a99ed019d..7d52aac7f9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.60 +2.4.0-foxtrot From bed42208b1856f1e02033ac483a0610fa7adb76e Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 5 Mar 2024 09:49:55 -0500 Subject: [PATCH 085/140] Add journald integration --- salt/elasticfleet/defaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index e4f54ceb0e..7b2d9d6a3c 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -65,6 +65,7 @@ elasticfleet: - http_endpoint - httpjson - iis + - journald - juniper - juniper_srx - kafka_log From 08f2b8251b95638ef5611e203bd476f8e259c74c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 5 Mar 2024 09:53:35 -0500 Subject: [PATCH 086/140] add GLOBALS.is_sensor --- salt/vars/globals.map.jinja | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/vars/globals.map.jinja b/salt/vars/globals.map.jinja index 6241732172..ed7129678a 100644 --- a/salt/vars/globals.map.jinja +++ b/salt/vars/globals.map.jinja @@ -8,6 +8,7 @@ set GLOBALS = { 'hostname': INIT.GRAINS.nodename, 'is_manager': false, + 'is_sensor': false, 'manager': INIT.GRAINS.master, 'minion_id': INIT.GRAINS.id, 'main_interface': INIT.PILLAR.host.mainint, @@ -63,5 +64,8 @@ {% do GLOBALS.update({'is_manager': true}) %} {% endif %} +{% if GLOBALS.role in GLOBALS.sensor_roles %} +{% do GLOBALS.update({'is_sensor': true}) %} +{% endif %} {% do salt['defaults.merge'](GLOBALS, ROLE_GLOBALS, merge_lists=False, in_place=True) %} From 1a58aa61a0409889194fe7f427078c7de5623aff Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 5 Mar 2024 09:54:40 -0500 Subject: [PATCH 087/140] only import pcap and suricata if sensor --- salt/sensoroni/files/sensoroni.json | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index 97c91f0b33..93708440a6 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -1,7 +1,5 @@ -{%- from 'vars/globals.map.jinja' import GLOBALS %} -{%- from 'sensoroni/map.jinja' import SENSORONIMERGED %} -{%- from 'pcap/config.map.jinja' import PCAPMERGED %} -{%- from 'suricata/map.jinja' import SURICATAMERGED %} +{% from 'vars/globals.map.jinja' import GLOBALS %} +{%- from 'sensoroni/map.jinja' import SENSORONIMERGED -%} { "logFilename": "/opt/sensoroni/logs/sensoroni.log", "logLevel":"info", @@ -24,24 +22,28 @@ "importer": {}, "statickeyauth": { "apiKey": "{{ GLOBALS.sensoroni_key }}" -{#- if PCAPMERGED.enabled is true then we know that steno is the pcap engine #} -{#- if it is false, then user has steno disabled in ui or has selected suricata for pcap engine #} -{%- if PCAPMERGED.enabled %} +{% if GLOBALS.is_sensor %} +{% from 'pcap/config.map.jinja' import PCAPMERGED %} +{% from 'suricata/map.jinja' import SURICATAMERGED %} +{# if PCAPMERGED.enabled is true then we know that steno is the pcap engine #} +{# if it is false, then user has steno disabled in ui or has selected suricata for pcap engine #} +{%- if PCAPMERGED.enabled %} }, "stenoquery": { "executablePath": "/opt/sensoroni/scripts/stenoquery.sh", "pcapInputPath": "/nsm/pcap", "pcapOutputPath": "/nsm/pcapout" } -{%- elif GLOBALS.pcap_engine == "SURICATA" and SURICATAMERGED.enabled %} +{%- elif GLOBALS.pcap_engine == "SURICATA" and SURICATAMERGED.enabled %} }, "suriquery": { "executablePath": "/opt/sensoroni/scripts/suriquery.sh", "pcapInputPath": "/nsm/suripcap", "pcapOutputPath": "/nsm/pcapout" } -{%- else %} +{%- else %} } +{% endif %} {%- endif %} } } From c0d19e11b9bad6ee1ebb93088f42f3e44b0bc13c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 5 Mar 2024 10:07:32 -0500 Subject: [PATCH 088/140] fix } placement --- salt/sensoroni/files/sensoroni.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index 93708440a6..5090967efb 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -41,9 +41,9 @@ "pcapInputPath": "/nsm/suripcap", "pcapOutputPath": "/nsm/pcapout" } -{%- else %} +{% endif %} +{%- else %} } -{% endif %} {%- endif %} } } From b9ebe6c40b2545bfb9c0d18e75d2be67c5f3d9f2 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 5 Mar 2024 12:58:34 -0500 Subject: [PATCH 089/140] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7d52aac7f9..5a99ed019d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.0-foxtrot +2.4.60 From 6eb608c3f53f2a9b6743d02eebe080c469343995 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 5 Mar 2024 15:05:03 -0500 Subject: [PATCH 090/140] Update so-minion --- salt/manager/tools/sbin/so-minion | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index d696e14c68..82c19e39b7 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -96,7 +96,7 @@ function pcapspace() { fi local s=$(( $SPACESIZE / 1000000 )) - local s1=$(( $s / 2 )) + local s1=$(( $s / 4 )) local s2=$(( $s1 / $lb_procs )) MAXPCAPFILES=$s2 From a686d46322ed335c8a7fd4220843e823511f2769 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 5 Mar 2024 15:09:02 -0500 Subject: [PATCH 091/140] Update so-minion --- salt/manager/tools/sbin/so-minion | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 82c19e39b7..7e33533b47 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -280,7 +280,7 @@ function add_sensor_to_minion() { echo " enabled: True" >> $PILLARFILE if [[ $is_pcaplimit ]]; then echo " config:" >> $PILLARFILE - echo " diskfreepercentage: 60" >> $PILLARFILE + echo " diskfreepercentage: 75" >> $PILLARFILE pcapspace fi echo " " >> $PILLARFILE From 4b5f00cef4d13a21ac8ff635a8ec625151e33e07 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 5 Mar 2024 16:42:20 -0500 Subject: [PATCH 092/140] fix oinkcodes with leading zeros --- salt/idstools/soc_idstools.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/idstools/soc_idstools.yaml b/salt/idstools/soc_idstools.yaml index f8ec3b8b67..3e3a68117d 100644 --- a/salt/idstools/soc_idstools.yaml +++ b/salt/idstools/soc_idstools.yaml @@ -6,6 +6,7 @@ idstools: description: Enter your registration code or oinkcode for paid NIDS rulesets. title: Registration Code global: True + forcedType: string helpLink: rules.html ruleset: description: 'Defines the ruleset you want to run. Options are ETOPEN or ETPRO. WARNING! Changing the ruleset will remove all existing Suricata rules of the previous ruleset and their associated overrides. This removal cannot be undone.' From 5687fdcf578eb44aa11f72646a11c284ea57838e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 5 Mar 2024 17:46:43 -0500 Subject: [PATCH 093/140] fix pcapspace function --- salt/manager/tools/sbin/so-minion | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index d696e14c68..54587774f3 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -84,8 +84,8 @@ function pcapspace() { local SPACESIZE=$(df -k /nsm | tail -1 | awk '{print $2}' | tr -d \n) else - local NSMSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/nsm"."1K-blocks" ') - local ROOTSIZE=$(salt '$MINION_ID' disk.usage --out=json | jq -r '.[]."/"."1K-blocks" ') + local NSMSIZE=$(salt "$MINION_ID" disk.usage --out=json | jq -r '.[]."/nsm"."1K-blocks" ') + local ROOTSIZE=$(salt "$MINION_ID" disk.usage --out=json | jq -r '.[]."/"."1K-blocks" ') if [[ "$NSMSIZE" == "null" ]]; then # Looks like there is no dedicated nsm partition. Using root @@ -97,7 +97,7 @@ function pcapspace() { local s=$(( $SPACESIZE / 1000000 )) local s1=$(( $s / 2 )) - local s2=$(( $s1 / $lb_procs )) + local s2=$(( $s1 / $CORECOUNT )) MAXPCAPFILES=$s2 From eaef076eba34104b5495aef002b8bab3e1b8422b Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Tue, 5 Mar 2024 17:52:24 -0500 Subject: [PATCH 094/140] Update so-minion --- salt/manager/tools/sbin/so-minion | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 6f037d3441..ab05fafcc6 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -96,8 +96,8 @@ function pcapspace() { fi local s=$(( $SPACESIZE / 1000000 )) - local s2=$(( $s1 / $CORECOUNT )) local s1=$(( $s / 4 )) + local s2=$(( $s1 / $CORECOUNT )) MAXPCAPFILES=$s2 From 1b47537a3f2d808e8cc9176fbfe636edadea6c7f Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 6 Mar 2024 07:16:50 -0500 Subject: [PATCH 095/140] Add Exclusion toggle --- salt/soc/defaults.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index d5ecbe71fb..60f2ee613b 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1201,6 +1201,9 @@ soc: - name: caseExcludeToggle filter: 'NOT _index:"*:so-case*"' enabled: true + - name: detectionsExcludeToggle + filter: 'NOT _index:"*:so-detection*"' + enabled: true - name: socExcludeToggle filter: 'NOT event.module:"soc"' enabled: true @@ -1471,6 +1474,9 @@ soc: - name: caseExcludeToggle filter: 'NOT _index:"*:so-case*"' enabled: true + - name: detectionsExcludeToggle + filter: 'NOT _index:"*:so-detection*"' + enabled: true - name: socExcludeToggle filter: 'NOT event.module:"soc"' enabled: true From 12653eec8c4565e9d2538c803fc9a8e53edfb1c6 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 6 Mar 2024 08:14:33 -0500 Subject: [PATCH 096/140] add new pcap annotations --- salt/sensoroni/defaults.yaml | 118 ++++++++++++++-------------- salt/sensoroni/files/sensoroni.json | 3 +- salt/sensoroni/soc_sensoroni.yaml | 5 ++ salt/soc/defaults.yaml | 1 + salt/soc/soc_soc.yaml | 3 + 5 files changed, 71 insertions(+), 59 deletions(-) diff --git a/salt/sensoroni/defaults.yaml b/salt/sensoroni/defaults.yaml index 4ebd666a94..7777985dd2 100644 --- a/salt/sensoroni/defaults.yaml +++ b/salt/sensoroni/defaults.yaml @@ -1,58 +1,60 @@ -sensoroni: - enabled: False - config: - analyze: - enabled: False - timeout_ms: 900000 - parallel_limit: 5 - node_checkin_interval_ms: 10000 - sensoronikey: - soc_host: - analyzers: - echotrail: - base_url: https://api.echotrail.io/insights/ - api_key: - elasticsearch: - base_url: - auth_user: - auth_pwd: - num_results: 10 - api_key: - index: _all - time_delta_minutes: 14400 - timestamp_field_name: '@timestamp' - map: {} - cert_path: - emailrep: - base_url: https://emailrep.io/ - api_key: - greynoise: - base_url: https://api.greynoise.io/ - api_key: - api_version: community - localfile: - file_path: [] - otx: - base_url: https://otx.alienvault.com/api/v1/ - api_key: - pulsedive: - base_url: https://pulsedive.com/api/ - api_key: - spamhaus: - lookup_host: zen.spamhaus.org - nameservers: [] - sublime_platform: - base_url: https://api.platform.sublimesecurity.com - api_key: - live_flow: False - mailbox_email_address: - message_source_id: - urlscan: - base_url: https://urlscan.io/api/v1/ - api_key: - enabled: False - visibility: public - timeout: 180 - virustotal: - base_url: https://www.virustotal.com/api/v3/search?query= - api_key: +sensoroni: + enabled: False + config: + analyze: + enabled: False + timeout_ms: 900000 + parallel_limit: 5 + node_checkin_interval_ms: 10000 + sensoronikey: + soc_host: + suripcap: + pcapMaxCount: 999999 + analyzers: + echotrail: + base_url: https://api.echotrail.io/insights/ + api_key: + elasticsearch: + base_url: + auth_user: + auth_pwd: + num_results: 10 + api_key: + index: _all + time_delta_minutes: 14400 + timestamp_field_name: '@timestamp' + map: {} + cert_path: + emailrep: + base_url: https://emailrep.io/ + api_key: + greynoise: + base_url: https://api.greynoise.io/ + api_key: + api_version: community + localfile: + file_path: [] + otx: + base_url: https://otx.alienvault.com/api/v1/ + api_key: + pulsedive: + base_url: https://pulsedive.com/api/ + api_key: + spamhaus: + lookup_host: zen.spamhaus.org + nameservers: [] + sublime_platform: + base_url: https://api.platform.sublimesecurity.com + api_key: + live_flow: False + mailbox_email_address: + message_source_id: + urlscan: + base_url: https://urlscan.io/api/v1/ + api_key: + enabled: False + visibility: public + timeout: 180 + virustotal: + base_url: https://www.virustotal.com/api/v3/search?query= + api_key: diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index 5090967efb..eb9c1131d8 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -39,7 +39,8 @@ "suriquery": { "executablePath": "/opt/sensoroni/scripts/suriquery.sh", "pcapInputPath": "/nsm/suripcap", - "pcapOutputPath": "/nsm/pcapout" + "pcapOutputPath": "/nsm/pcapout", + "pcapMaxCount": {{ SENSORONIMERGED.config.suripcap.pcapMaxCount }} } {% endif %} {%- else %} diff --git a/salt/sensoroni/soc_sensoroni.yaml b/salt/sensoroni/soc_sensoroni.yaml index f7a10c6f7a..7b8495dc58 100644 --- a/salt/sensoroni/soc_sensoroni.yaml +++ b/salt/sensoroni/soc_sensoroni.yaml @@ -37,6 +37,11 @@ sensoroni: helpLink: grid.html global: True advanced: True + suripcap: + pcapMaxCount: + description: The maximum number of PCAP packets to extract from eligible PCAP files, for PCAP jobs. If there are issues fetching excessively large packet streams consider lowering this value to reduce the number of collected packets returned to the user interface. + helpLink: sensoroni.html + advanced: True analyzers: echotrail: api_key: diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index d5ecbe71fb..abcd123087 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1103,6 +1103,7 @@ soc: esSearchOffsetMs: 1800000 maxLogLength: 1024 asyncThreshold: 10 + lookupTunnelParent: true influxdb: hostUrl: token: diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index e80e98a7e5..645a4c8a9d 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -126,6 +126,9 @@ soc: description: Maximum number of events that can be acknowledged synchronously. When acknowledging large numbers of events, where the count exceeds this value, the acknowledge update will be performed in the background, as it can take several minutes to complete. global: True advanced: True + lookupTunnelParent: + description: When true, if a pivoted event appears to be encapsulated, such as in a VxLan packet, then SOC will pivot to the VxLan packet stream. This can be useful if the PCAP parser is unable to locate the encapsulated packets. However, if the parser is written in a way that it can find a given filter even if its encapsulated, this is best left to false, as the analyst will see the intended packet data rather than the wrapping packet data. + global: True sostatus: refreshIntervalMs: description: Duration (in milliseconds) between refreshes of the grid status. Shortening this duration may not have expected results, as the backend systems feeding this sostatus data will continue their updates as scheduled. From 0f12297f5019e91def5af525ffcd113108dc9e29 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 6 Mar 2024 08:19:42 -0500 Subject: [PATCH 097/140] add new pcap annotations --- salt/soc/soc_soc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 645a4c8a9d..799d9af4ee 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -127,7 +127,7 @@ soc: global: True advanced: True lookupTunnelParent: - description: When true, if a pivoted event appears to be encapsulated, such as in a VxLan packet, then SOC will pivot to the VxLan packet stream. This can be useful if the PCAP parser is unable to locate the encapsulated packets. However, if the parser is written in a way that it can find a given filter even if its encapsulated, this is best left to false, as the analyst will see the intended packet data rather than the wrapping packet data. + description: When true, if a pivoted event appears to be encapsulated, such as in a VXLAN packet, then SOC will pivot to the VXLAN packet stream. When false, SOC will attempt to pivot to the encapsulated packet stream itself, but at the risk that it may be unable to locate it in the stored PCAP data. global: True sostatus: refreshIntervalMs: From f58c104d899bf99fa1d3b566797eb2447a909917 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 6 Mar 2024 09:51:56 -0500 Subject: [PATCH 098/140] Update so-minion --- salt/manager/tools/sbin/so-minion | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index ab05fafcc6..09708707f7 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -81,6 +81,8 @@ function getinstallinfo() { function pcapspace() { if [[ "$OPERATION" == "setup" ]]; then + # Use 25% for PCAP + PCAP_PERCENTAGE=1 local SPACESIZE=$(df -k /nsm | tail -1 | awk '{print $2}' | tr -d \n) else @@ -96,10 +98,9 @@ function pcapspace() { fi local s=$(( $SPACESIZE / 1000000 )) - local s1=$(( $s / 4 )) - local s2=$(( $s1 / $CORECOUNT )) + local s1=$(( $s / 4 * $PCAP_PERCENTAGE )) - MAXPCAPFILES=$s2 + MAX_PCAP_SPACE=$s1 } @@ -271,17 +272,12 @@ function add_sensor_to_minion() { echo " config:" >> $PILLARFILE echo " af-packet:" >> $PILLARFILE echo " threads: '$CORECOUNT'" >> $PILLARFILE - if [[ $is_pcaplimit ]]; then - echo " output:" >> $PILLARFILE - echo " pcap-log:" >> $PILLARFILE - echo " max-files: '$MAXPCAPFILES'" >> $PILLARFILE - fi echo "pcap:" >> $PILLARFILE echo " enabled: True" >> $PILLARFILE if [[ $is_pcaplimit ]]; then echo " config:" >> $PILLARFILE echo " diskfreepercentage: 75" >> $PILLARFILE - pcapspace + echo " suripcapmaxsize: $MAX_PCAP_SPACE" >> $PILLARFILE fi echo " " >> $PILLARFILE } @@ -563,6 +559,7 @@ function createIDH() { function createHEAVYNODE() { is_pcaplimit=true + PCAP_PERCENTAGE=1 pcapspace add_elasticsearch_to_minion add_elastic_agent_to_minion @@ -575,6 +572,7 @@ function createHEAVYNODE() { function createSENSOR() { is_pcaplimit=true + PCAP_PERCENTAGE=3 pcapspace add_sensor_to_minion add_strelka_to_minion From a63fca727ce144a90d6fd01a82707ecfcf32a39f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 6 Mar 2024 10:02:06 -0500 Subject: [PATCH 099/140] Update soc_suricata.yaml --- salt/suricata/soc_suricata.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index 4f9a80d860..fbd6e84ee1 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -188,17 +188,11 @@ suricata: advanced: True readonly: True helpLink: suricata.html - limit: - description: File size limit per thread. To determine max PCAP size multiple threads x max-files x limit. - helpLink: suricata.html mode: description: Suricata PCAP mode. Currently only multi is supported. advanced: True readonly: True helpLink: suricata.html - max-files: - description: Max PCAP files per thread. To determine max PCAP size multiple threads x max-files x limit. - helpLink: suricata.html use-stream-depth: description: Set to "no" to ignore the stream depth and capture the entire flow. Set this to "yes" to truncate the flow based on the stream depth. advanced: True From f836d6a61d8e650f1d02e4542459f1b4989afb03 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 6 Mar 2024 10:06:17 -0500 Subject: [PATCH 100/140] Update so-minion From 4dfa1a5626387ab70c389565fa74a542cc44949b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 6 Mar 2024 10:35:10 -0500 Subject: [PATCH 101/140] Move Suricata around --- salt/suricata/defaults.yaml | 5 +++-- salt/suricata/soc_suricata.yaml | 8 ++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml index eb2c181e32..42af3fc556 100644 --- a/salt/suricata/defaults.yaml +++ b/salt/suricata/defaults.yaml @@ -1,5 +1,8 @@ suricata: enabled: False + pcap: + filesize: 1000mb + maxsize: 25 config: threading: set-cpu-affinity: "no" @@ -132,9 +135,7 @@ suricata: lz4-checksum: "no" lz4-level: 8 filename: "%n/so-pcap.%t" - limit: "1000mb" mode: "multi" - max-files: 10 use-stream-depth: "no" conditional: "all" dir: "/nsm/suripcap" diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index fbd6e84ee1..88b460af8f 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -19,6 +19,14 @@ suricata: multiline: True title: Classifications helpLink: suricata.html + pcap: + filesize: + description: Max file size for individual PCAP files written by Suricata. Increasing this number could improve write performance at the expense of pcap retrieval times. + advanced: True + helplink: suricata.html + maxsize: + description: Size in GB for total usage size of PCAP on disk. + helplink: suricata.html config: af-packet: interface: From 167aff24f61b8bbcaced10e05f8dfde3dd4acf0a Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 6 Mar 2024 11:03:52 -0500 Subject: [PATCH 102/140] detections annotations --- salt/soc/defaults.yaml | 1 + salt/soc/soc_soc.yaml | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index abcd123087..ad1f8bb5f2 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1147,6 +1147,7 @@ soc: tipTimeoutMs: 6000 cacheExpirationMs: 300000 casesEnabled: true + detectionsEnabled: false inactiveTools: ['toolUnused'] tools: - name: toolKibana diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 799d9af4ee..08a29766dd 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -79,11 +79,11 @@ soc: modules: elastalertengine: sigmaRulePackages: - description: 'Defines the Sigma Community Ruleset you want to run. One of these (core | core+ | core++ | all ) as well as an optional Add-on (emerging_threats_addon). WARNING! Changing the ruleset will remove all existing Sigma rules of the previous ruleset and their associated overrides. This removal cannot be undone.' + description: 'Defines the Sigma Community Ruleset you want to run. One of these (core | core+ | core++ | all ) as well as an optional Add-on (emerging_threats_addon). WARNING! Changing the ruleset will remove all existing Sigma rules of the previous ruleset and their associated overrides. This removal cannot be undone. (future use, not yet complete)' global: True advanced: False autoUpdateEnabled: - description: 'Set to true to enable automatic Internet-connected updates of the Sigma Community Ruleset. If this is an Airgap system, this setting will be overridden and set to false.' + description: 'Set to true to enable automatic Internet-connected updates of the Sigma Community Ruleset. If this is an Airgap system, this setting will be overridden and set to false. (future use, not yet complete)' global: True advanced: True elastic: @@ -149,7 +149,7 @@ soc: advanced: True strelkaengine: autoUpdateEnabled: - description: 'Set to true to enable automatic Internet-connected updates of the Yara rulesets. If this is an Airgap system, this setting will be overridden and set to false.' + description: 'Set to true to enable automatic Internet-connected updates of the Yara rulesets. If this is an Airgap system, this setting will be overridden and set to false. (future use, not yet complete)' global: True advanced: True client: @@ -174,6 +174,9 @@ soc: casesEnabled: description: Set to true to enable case management in SOC. global: True + detectionsEnabled: + description: Set to true to enable the Detections module in SOC. (future use, not yet complete) + global: True inactiveTools: description: List of external tools to remove from the SOC UI. global: True From ad120934295999845436cdcb303ac3ca51c621e6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 6 Mar 2024 11:05:06 -0500 Subject: [PATCH 103/140] Fix percent calc --- salt/manager/tools/sbin/so-minion | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 09708707f7..a3d8230b55 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -83,6 +83,7 @@ function pcapspace() { if [[ "$OPERATION" == "setup" ]]; then # Use 25% for PCAP PCAP_PERCENTAGE=1 + DFREEPERCENT=75 local SPACESIZE=$(df -k /nsm | tail -1 | awk '{print $2}' | tr -d \n) else @@ -269,6 +270,10 @@ function add_sensor_to_minion() { echo " lb_procs: '$CORECOUNT'" >> $PILLARFILE echo "suricata:" >> $PILLARFILE echo " enabled: True " >> $PILLARFILE + if [[ $is_pcaplimit ]]; then + echo " pcap:" >> $PILLARFILE + echo " maxsize: $MAX_PCAP_SPACE" >> $PILLARFILE + fi echo " config:" >> $PILLARFILE echo " af-packet:" >> $PILLARFILE echo " threads: '$CORECOUNT'" >> $PILLARFILE @@ -276,8 +281,7 @@ function add_sensor_to_minion() { echo " enabled: True" >> $PILLARFILE if [[ $is_pcaplimit ]]; then echo " config:" >> $PILLARFILE - echo " diskfreepercentage: 75" >> $PILLARFILE - echo " suripcapmaxsize: $MAX_PCAP_SPACE" >> $PILLARFILE + echo " diskfreepercentage: $DFREEPERCENT" >> $PILLARFILE fi echo " " >> $PILLARFILE } @@ -560,6 +564,7 @@ function createIDH() { function createHEAVYNODE() { is_pcaplimit=true PCAP_PERCENTAGE=1 + DFREEPERCENT=75 pcapspace add_elasticsearch_to_minion add_elastic_agent_to_minion @@ -572,6 +577,7 @@ function createHEAVYNODE() { function createSENSOR() { is_pcaplimit=true + DFREEPERCENT=10 PCAP_PERCENTAGE=3 pcapspace add_sensor_to_minion From 1cbac11fae137961e67155122856a816989c8a60 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 6 Mar 2024 11:08:03 -0500 Subject: [PATCH 104/140] detections annotations --- salt/soc/soc_soc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 08a29766dd..cb939f7581 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -34,7 +34,7 @@ soc: helpLink: soc-customization.html sigma_final_pipeline__yaml: title: Final Sigma Pipeline - description: Final Processing Pipeline for Sigma Rules + description: Final Processing Pipeline for Sigma Rules (future use, not yet complete) syntax: yaml file: True global: True From 9a413a2e3189aa24bc654c9ebe388e16153b898f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 6 Mar 2024 12:42:22 -0500 Subject: [PATCH 105/140] Fix location of repo --- setup/so-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index ef1df4a719..4aae0f5bd8 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1811,8 +1811,8 @@ repo_sync_local() { info "Adding Repo Download Configuration" mkdir -p /nsm/repo mkdir -p /opt/so/conf/reposync/cache - echo "https://repo.securityonion.net/file/so-repo/prod/2.4/oracle/9.3" > /opt/so/conf/reposync/mirror.txt - echo "https://repo-alt.securityonion.net/prod/2.4/oracle/9.3" >> /opt/so/conf/reposync/mirror.txt + echo "https://repo.securityonion.net/file/so-repo/prod/2.4/oracle/9" > /opt/so/conf/reposync/mirror.txt + echo "https://repo-alt.securityonion.net/prod/2.4/oracle/9" >> /opt/so/conf/reposync/mirror.txt echo "[main]" > /opt/so/conf/reposync/repodownload.conf echo "gpgcheck=1" >> /opt/so/conf/reposync/repodownload.conf echo "installonly_limit=3" >> /opt/so/conf/reposync/repodownload.conf From 7f1e786e3d1e75297ecb7bda12a63a60d6521904 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 6 Mar 2024 12:56:09 -0500 Subject: [PATCH 106/140] Consolidate PCAP settings --- salt/suricata/defaults.yaml | 16 +++---- salt/suricata/soc_suricata.yaml | 84 ++++++++++++++++----------------- 2 files changed, 50 insertions(+), 50 deletions(-) diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml index 42af3fc556..0252d3a813 100644 --- a/salt/suricata/defaults.yaml +++ b/salt/suricata/defaults.yaml @@ -3,6 +3,14 @@ suricata: pcap: filesize: 1000mb maxsize: 25 + compression: "none" + lz4-checksum: "no" + lz4-level: 8 + filename: "%n/so-pcap.%t" + mode: "multi" + use-stream-depth: "no" + conditional: "all" + dir: "/nsm/suripcap" config: threading: set-cpu-affinity: "no" @@ -131,14 +139,6 @@ suricata: enabled: "no" pcap-log: enabled: "no" - compression: "none" - lz4-checksum: "no" - lz4-level: 8 - filename: "%n/so-pcap.%t" - mode: "multi" - use-stream-depth: "no" - conditional: "all" - dir: "/nsm/suripcap" alert-debug: enabled: "no" alert-prelude: diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index 88b460af8f..da7586e97b 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -27,6 +27,47 @@ suricata: maxsize: description: Size in GB for total usage size of PCAP on disk. helplink: suricata.html + compression: + description: Enable compression of Suricata PCAP. Currently unsupported + advanced: True + readonly: True + helpLink: suricata.html + lz4-checksum: + description: Enable PCAP lz4 checksum. Currently unsupported + advanced: True + readonly: True + helpLink: suricata.html + lz4-level: + description: lz4 compression level of PCAP. 0 for no compression 16 for max compression. Currently unsupported + advanced: True + readonly: True + helpLink: suricata.html + filename: + description: Filename output for Suricata PCAP. + advanced: True + readonly: True + helpLink: suricata.html + mode: + description: Suricata PCAP mode. Currently only multi is supported. + advanced: True + readonly: True + helpLink: suricata.html + use-stream-depth: + description: Set to "no" to ignore the stream depth and capture the entire flow. Set this to "yes" to truncate the flow based on the stream depth. + advanced: True + regex: ^(yes|no)$ + regexFailureMessage: You must enter either yes or no. + helpLink: suricata.html + conditional: + description: Set to "all" to capture PCAP for all flows. Set to "alert" to capture PCAP just for alerts or set to "tag" to capture PCAP for just tagged rules. + regex: ^(all|alert|tag)$ + regexFailureMessage: You must enter either all, alert or tag. + helpLink: suricata.html + dir: + description: Parent directory to store PCAP. + advanced: True + readonly: True + helpLink: suricata.html config: af-packet: interface: @@ -175,48 +216,7 @@ suricata: description: This value is ignored by SO. pcapengine in globals takes precidence. readonly: True helpLink: suricata.html - advanced: True - compression: - description: Enable compression of Suricata PCAP. Currently unsupported - advanced: True - readonly: True - helpLink: suricata.html - lz4-checksum: - description: Enable PCAP lz4 checksum. Currently unsupported - advanced: True - readonly: True - helpLink: suricata.html - lz4-level: - description: lz4 compression level of PCAP. 0 for no compression 16 for max compression. Currently unsupported - advanced: True - readonly: True - helpLink: suricata.html - filename: - description: Filename output for Suricata PCAP. - advanced: True - readonly: True - helpLink: suricata.html - mode: - description: Suricata PCAP mode. Currently only multi is supported. - advanced: True - readonly: True - helpLink: suricata.html - use-stream-depth: - description: Set to "no" to ignore the stream depth and capture the entire flow. Set this to "yes" to truncate the flow based on the stream depth. - advanced: True - regex: ^(yes|no)$ - regexFailureMessage: You must enter either yes or no. - helpLink: suricata.html - conditional: - description: Set to "all" to capture PCAP for all flows. Set to "alert" to capture PCAP just for alerts or set to "tag" to capture PCAP for just tagged rules. - regex: ^(all|alert|tag)$ - regexFailureMessage: You must enter either all, alert or tag. - helpLink: suricata.html - dir: - description: Parent directory to store PCAP. - advanced: True - readonly: True - helpLink: suricata.html + advanced: True asn1-max-frames: description: Maximum nuber of asn1 frames to decode. helpLink: suricata.html From cf232534ca9a1609da8946adc24dd5f7116cfc24 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 6 Mar 2024 14:42:07 -0500 Subject: [PATCH 107/140] move suricata.pcap to suricata.config.outputs.pcap-log --- salt/suricata/map.jinja | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/salt/suricata/map.jinja b/salt/suricata/map.jinja index 5f6e913f50..d8ce271fed 100644 --- a/salt/suricata/map.jinja +++ b/salt/suricata/map.jinja @@ -63,6 +63,18 @@ {# before we change outputs back to list, enable pcap-log if suricata is the pcapengine #} {% if GLOBALS.pcap_engine == "SURICATA" %} {% do SURICATAMERGED.config.outputs['pcap-log'].update({'enabled': 'yes'}) %} +{# move the items in suricata.pcap into suricata.config.outputs.pcap-log. these items were placed under suricata.config for ease of access in SOC #} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'compression': SURICATAMERGED.pcap.compression}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'lz4-checksum': SURICATAMERGED.pcap['lz4-checksum']}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'lz4-level': SURICATAMERGED.pcap['lz4-level']}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'filename': SURICATAMERGED.pcap.filename}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'limit': SURICATAMERGED.pcap.filesize}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'mode': SURICATAMERGED.pcap.mode}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'max-stream-depth': SURICATAMERGED.pcap['max-stream-depth']}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'conditional': SURICATAMERGED.pcap.conditional}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'dir': SURICATAMERGED.pcap.dir}) %} +{% set maxfiles = (SURICATAMERGED.pcap.maxsize / SURICATAMERGED.pcap.filesize) | round | int %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'max-files': maxfiles}) %} {% endif %} {# outputs is a list but we convert to dict in defaults to work with ui #} From 583227290f1fedc68ffb9abf14b2b92c8677171d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 6 Mar 2024 15:18:22 -0500 Subject: [PATCH 108/140] fix max-files calc --- salt/suricata/map.jinja | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/suricata/map.jinja b/salt/suricata/map.jinja index d8ce271fed..77cdbe7c59 100644 --- a/salt/suricata/map.jinja +++ b/salt/suricata/map.jinja @@ -70,10 +70,11 @@ {% do SURICATAMERGED.config.outputs['pcap-log'].update({'filename': SURICATAMERGED.pcap.filename}) %} {% do SURICATAMERGED.config.outputs['pcap-log'].update({'limit': SURICATAMERGED.pcap.filesize}) %} {% do SURICATAMERGED.config.outputs['pcap-log'].update({'mode': SURICATAMERGED.pcap.mode}) %} -{% do SURICATAMERGED.config.outputs['pcap-log'].update({'max-stream-depth': SURICATAMERGED.pcap['max-stream-depth']}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'use-stream-depth': SURICATAMERGED.pcap['use-stream-depth']}) %} {% do SURICATAMERGED.config.outputs['pcap-log'].update({'conditional': SURICATAMERGED.pcap.conditional}) %} {% do SURICATAMERGED.config.outputs['pcap-log'].update({'dir': SURICATAMERGED.pcap.dir}) %} -{% set maxfiles = (SURICATAMERGED.pcap.maxsize / SURICATAMERGED.pcap.filesize) | round | int %} +{# multiply maxsize by 1000 since it is saved in GB, i.e. 52 = 52000MB. filesize is also saved in MB and we strip the MB and convert to int #} +{% set maxfiles = (SURICATAMERGED.pcap.maxsize * 1000 / SURICATAMERGED.pcap.filesize[:-2] | int) | round | int %} {% do SURICATAMERGED.config.outputs['pcap-log'].update({'max-files': maxfiles}) %} {% endif %} From 17a75d5bd251e758706e53380f29d89930eed997 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 6 Mar 2024 17:19:01 -0500 Subject: [PATCH 109/140] Run stig post remediate scan against default ol9 scap-security-guide. Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/stig/enabled.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/stig/enabled.sls b/salt/stig/enabled.sls index 0f347f8bcd..c35c91a552 100644 --- a/salt/stig/enabled.sls +++ b/salt/stig/enabled.sls @@ -85,7 +85,7 @@ remove_nullok_from_system_auth_auth: run_post_scan: cmd.run: - - name: 'oscap xccdf eval --profile {{ OSCAP_PROFILE_NAME }} --results {{ OSCAP_OUTPUT_DIR }}/post-oscap-results.xml --report {{ OSCAP_OUTPUT_DIR }}/post-oscap-report.html {{ OSCAP_PROFILE_LOCATION }}' + - name: 'oscap xccdf eval --profile {{ OSCAP_PROFILE_NAME }} --results {{ OSCAP_OUTPUT_DIR }}/post-oscap-results.xml --report {{ OSCAP_OUTPUT_DIR }}/post-oscap-report.html /usr/share/xml/scap/ssg/content/ssg-ol9-ds.xml' - success_retcodes: - 2 From 70f3ce0536b4c4aa1c7b24bbea28968ec5fb7c28 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 6 Mar 2024 17:32:06 -0500 Subject: [PATCH 110/140] change how maxfiles is calculated --- salt/suricata/map.jinja | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/salt/suricata/map.jinja b/salt/suricata/map.jinja index 77cdbe7c59..6ba3c3b734 100644 --- a/salt/suricata/map.jinja +++ b/salt/suricata/map.jinja @@ -8,6 +8,24 @@ {% set surimeta_evelog_index = [] %} {% set surimeta_filestore_index = [] %} +{# before we change outputs back to list, enable pcap-log if suricata is the pcapengine #} +{% if GLOBALS.pcap_engine == "SURICATA" %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'enabled': 'yes'}) %} +{# move the items in suricata.pcap into suricata.config.outputs.pcap-log. these items were placed under suricata.config for ease of access in SOC #} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'compression': SURICATAMERGED.pcap.compression}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'lz4-checksum': SURICATAMERGED.pcap['lz4-checksum']}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'lz4-level': SURICATAMERGED.pcap['lz4-level']}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'filename': SURICATAMERGED.pcap.filename}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'limit': SURICATAMERGED.pcap.filesize}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'mode': SURICATAMERGED.pcap.mode}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'use-stream-depth': SURICATAMERGED.pcap['use-stream-depth']}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'conditional': SURICATAMERGED.pcap.conditional}) %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'dir': SURICATAMERGED.pcap.dir}) %} +{# multiply maxsize by 1000 since it is saved in GB, i.e. 52 = 52000MB. filesize is also saved in MB and we strip the MB and convert to int #} +{% set maxfiles = (SURICATAMERGED.pcap.maxsize * 1000 / (SURICATAMERGED.pcap.filesize[:-2] | int) / SURICATAMERGED.config['af-packet'].threads | int) | round | int %} +{% do SURICATAMERGED.config.outputs['pcap-log'].update({'max-files': maxfiles}) %} +{% endif %} + {# suricata.config.af-packet has to be rewritten here since we cant display '- interface' in the ui #} {# we are limited to only one iterface #} {% load_yaml as afpacket %} @@ -60,24 +78,6 @@ {% do SURICATAMERGED.config.outputs['file-store'].update({'enabled':suricata_mdengine.suricata.config.outputs[surimeta_filestore_index]['file-store']['enabled']}) %} {% endif %} -{# before we change outputs back to list, enable pcap-log if suricata is the pcapengine #} -{% if GLOBALS.pcap_engine == "SURICATA" %} -{% do SURICATAMERGED.config.outputs['pcap-log'].update({'enabled': 'yes'}) %} -{# move the items in suricata.pcap into suricata.config.outputs.pcap-log. these items were placed under suricata.config for ease of access in SOC #} -{% do SURICATAMERGED.config.outputs['pcap-log'].update({'compression': SURICATAMERGED.pcap.compression}) %} -{% do SURICATAMERGED.config.outputs['pcap-log'].update({'lz4-checksum': SURICATAMERGED.pcap['lz4-checksum']}) %} -{% do SURICATAMERGED.config.outputs['pcap-log'].update({'lz4-level': SURICATAMERGED.pcap['lz4-level']}) %} -{% do SURICATAMERGED.config.outputs['pcap-log'].update({'filename': SURICATAMERGED.pcap.filename}) %} -{% do SURICATAMERGED.config.outputs['pcap-log'].update({'limit': SURICATAMERGED.pcap.filesize}) %} -{% do SURICATAMERGED.config.outputs['pcap-log'].update({'mode': SURICATAMERGED.pcap.mode}) %} -{% do SURICATAMERGED.config.outputs['pcap-log'].update({'use-stream-depth': SURICATAMERGED.pcap['use-stream-depth']}) %} -{% do SURICATAMERGED.config.outputs['pcap-log'].update({'conditional': SURICATAMERGED.pcap.conditional}) %} -{% do SURICATAMERGED.config.outputs['pcap-log'].update({'dir': SURICATAMERGED.pcap.dir}) %} -{# multiply maxsize by 1000 since it is saved in GB, i.e. 52 = 52000MB. filesize is also saved in MB and we strip the MB and convert to int #} -{% set maxfiles = (SURICATAMERGED.pcap.maxsize * 1000 / SURICATAMERGED.pcap.filesize[:-2] | int) | round | int %} -{% do SURICATAMERGED.config.outputs['pcap-log'].update({'max-files': maxfiles}) %} -{% endif %} - {# outputs is a list but we convert to dict in defaults to work with ui #} {# below they are converted back to lists #} {% load_yaml as outputs %} From 005930f7fd70577ced8bec4f25650ec42fb1eccd Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 7 Mar 2024 15:41:23 +0000 Subject: [PATCH 111/140] Add error.message mapping for system.syslog --- .../logs-system.syslog@custom.json | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-system.syslog@custom.json diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-system.syslog@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs-system.syslog@custom.json new file mode 100644 index 0000000000..0123fb9561 --- /dev/null +++ b/salt/elasticsearch/templates/component/elastic-agent/logs-system.syslog@custom.json @@ -0,0 +1,22 @@ +{ + "template": { + "mappings": { + "properties": { + "error": { + "properties": { + "message": { + "type": "match_only_text" + } + } + } + } + } + }, + "_meta": { + "package": { + "name": "system" + }, + "managed_by": "fleet", + "managed": true + } +} From fffef9b621d9afff1d6fcdae17c8a0fab76aadc5 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 7 Mar 2024 12:31:51 -0500 Subject: [PATCH 112/140] gracefully handle status check failure on ubuntu --- salt/common/tools/sbin/so-common-status-check | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-common-status-check b/salt/common/tools/sbin/so-common-status-check index d713ba6c60..625e0f1997 100644 --- a/salt/common/tools/sbin/so-common-status-check +++ b/salt/common/tools/sbin/so-common-status-check @@ -47,10 +47,14 @@ def check_for_fps(): fps = 1 except FileNotFoundError: fn = '/proc/sys/crypto/' + feat_full + '_enabled' - with open(fn, 'r') as f: - contents = f.read() - if '1' in contents: - fps = 1 + try: + with open(fn, 'r') as f: + contents = f.read() + if '1' in contents: + fps = 1 + except: + # Unknown, so assume 0 + fps = 0 with open('/opt/so/log/sostatus/lks_enabled', 'w') as f: f.write(str(fps)) From 40574982e4afbf458a60d77267f1b2fde460fee9 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 7 Mar 2024 14:25:43 -0500 Subject: [PATCH 113/140] unswap files --- salt/common/tools/sbin/so-common-status-check | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-common-status-check b/salt/common/tools/sbin/so-common-status-check index 625e0f1997..4aa9811239 100644 --- a/salt/common/tools/sbin/so-common-status-check +++ b/salt/common/tools/sbin/so-common-status-check @@ -56,7 +56,7 @@ def check_for_fps(): # Unknown, so assume 0 fps = 0 - with open('/opt/so/log/sostatus/lks_enabled', 'w') as f: + with open('/opt/so/log/sostatus/fps_enabled', 'w') as f: f.write(str(fps)) def check_for_lks(): @@ -80,7 +80,7 @@ def check_for_lks(): lks = 1 if lks: break - with open('/opt/so/log/sostatus/fps_enabled', 'w') as f: + with open('/opt/so/log/sostatus/lks_enabled', 'w') as f: f.write(str(lks)) def fail(msg): From 06257b9c4a0c046ca4d44ca9b1a301f799a92ec8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 7 Mar 2024 14:32:46 -0500 Subject: [PATCH 114/140] Update so-minion --- salt/manager/tools/sbin/so-minion | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index a3d8230b55..cb4e40adee 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -83,7 +83,7 @@ function pcapspace() { if [[ "$OPERATION" == "setup" ]]; then # Use 25% for PCAP PCAP_PERCENTAGE=1 - DFREEPERCENT=75 + DFREEPERCENT=21 local SPACESIZE=$(df -k /nsm | tail -1 | awk '{print $2}' | tr -d \n) else @@ -564,7 +564,7 @@ function createIDH() { function createHEAVYNODE() { is_pcaplimit=true PCAP_PERCENTAGE=1 - DFREEPERCENT=75 + DFREEPERCENT=21 pcapspace add_elasticsearch_to_minion add_elastic_agent_to_minion From 3eb6fe2df97b76059ec9876f082061773ec4c71b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 8 Mar 2024 09:52:12 -0500 Subject: [PATCH 115/140] allow managersearch to receiver redis and 5644 --- salt/firewall/defaults.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 75df49b253..75a70828e4 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -1295,6 +1295,10 @@ firewall: portgroups: - redis - beats_5644 + managersearch: + portgroups: + - redis + - beats_5644 self: portgroups: - redis From 4e329359919367869ff8a0d3032e8823386863af Mon Sep 17 00:00:00 2001 From: Wes Date: Fri, 8 Mar 2024 16:24:37 +0000 Subject: [PATCH 116/140] Add Strelka config back --- salt/manager/init.sls | 45 +++++++++++++++++++++++++++++++++ salt/strelka/backend/config.sls | 10 ++++++++ 2 files changed, 55 insertions(+) diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 51590a6ec4..56e72c279e 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -103,6 +103,51 @@ rules_dir: - group: socore - makedirs: True +{% if STRELKAMERGED.rules.enabled %} + strelkarepos: + file.managed: + - name: /opt/so/conf/strelka/repos.txt + - source: salt://strelka/rules/repos.txt.jinja + - template: jinja + - defaults: + STRELKAREPOS: {{ STRELKAMERGED.rules.repos }} + - makedirs: True + strelka-yara-update: + {% if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} + cron.present: + {% else %} + cron.absent: + {% endif %} + - user: socore + - name: '/usr/sbin/so-yara-update >> /opt/so/log/yarasync/yara-update.log 2>&1' + - identifier: strelka-yara-update + - hour: '7' + - minute: '1' + strelka-yara-download: + {% if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} + cron.present: + {% else %} + cron.absent: + {% endif %} + - user: socore + - name: '/usr/sbin/so-yara-download >> /opt/so/log/yarasync/yara-download.log 2>&1' + - identifier: strelka-yara-download + - hour: '7' + - minute: '1' + {% if not GLOBALS.airgap %} + update_yara_rules: + cmd.run: + - name: /usr/sbin/so-yara-update + - onchanges: + - file: yara_update_scripts + download_yara_rules: + cmd.run: + - name: /usr/sbin/so-yara-download + - onchanges: + - file: yara_update_scripts + {% endif %} + {% endif %} + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/strelka/backend/config.sls b/salt/strelka/backend/config.sls index b39e06ac83..db18a68ccf 100644 --- a/salt/strelka/backend/config.sls +++ b/salt/strelka/backend/config.sls @@ -50,6 +50,16 @@ backend_taste: - user: 939 - group: 939 +{% if STRELKAMERGED.rules.enabled %} +strelkarules: + file.recurse: + - name: /opt/so/conf/strelka/rules + - source: salt://strelka/rules + - user: 939 + - group: 939 + - clean: True +{% endif %} + {% else %} {{sls}}_state_not_allowed: From fc66a549027070ab75eecd9fda9d7fbb7aee074a Mon Sep 17 00:00:00 2001 From: Wes Date: Fri, 8 Mar 2024 16:26:14 +0000 Subject: [PATCH 117/140] Add Strelka download and update scripts back --- .../manager/tools/sbin_jinja/so-yara-download | 51 +++++++++++++++++++ salt/manager/tools/sbin_jinja/so-yara-update | 41 +++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 salt/manager/tools/sbin_jinja/so-yara-download create mode 100644 salt/manager/tools/sbin_jinja/so-yara-update diff --git a/salt/manager/tools/sbin_jinja/so-yara-download b/salt/manager/tools/sbin_jinja/so-yara-download new file mode 100644 index 0000000000..aa95762539 --- /dev/null +++ b/salt/manager/tools/sbin_jinja/so-yara-download @@ -0,0 +1,51 @@ +#!/bin/bash +NOROOT=1 +. /usr/sbin/so-common + +{%- set proxy = salt['pillar.get']('manager:proxy') %} +{%- set noproxy = salt['pillar.get']('manager:no_proxy', '') %} + +# Download the rules from the internet +{%- if proxy %} +export http_proxy={{ proxy }} +export https_proxy={{ proxy }} +export no_proxy="{{ noproxy }}" +{%- endif %} + +repos="/opt/so/conf/strelka/repos.txt" +output_dir=/nsm/rules/yara +gh_status=$(curl -s -o /dev/null -w "%{http_code}" https://github.com) +clone_dir="/tmp" +if [ "$gh_status" == "200" ] || [ "$gh_status" == "301" ]; then + + while IFS= read -r repo; do + if ! $(echo "$repo" | grep -qE '^#'); then + # Remove old repo if existing bc of previous error condition or unexpected disruption + repo_name=`echo $repo | awk -F '/' '{print $NF}'` + [ -d $output_dir/$repo_name ] && rm -rf $output_dir/$repo_name + + # Clone repo and make appropriate directories for rules + git clone $repo $clone_dir/$repo_name + echo "Analyzing rules from $clone_dir/$repo_name..." + mkdir -p $output_dir/$repo_name + # Ensure a copy of the license is available for the rules + [ -f $clone_dir/$repo_name/LICENSE ] && cp $clone_dir/$repo_name/LICENSE $output_dir/$repo_name + + # Copy over rules + for i in $(find $clone_dir/$repo_name -name "*.yar*"); do + rule_name=$(echo $i | awk -F '/' '{print $NF}') + cp $i $output_dir/$repo_name + done + rm -rf $clone_dir/$repo_name + fi + done < $repos + + echo "Done!" + +/usr/sbin/so-yara-update + +else + echo "Server returned $gh_status status code." + echo "No connectivity to Github...exiting..." + exit 1 +fi diff --git a/salt/manager/tools/sbin_jinja/so-yara-update b/salt/manager/tools/sbin_jinja/so-yara-update new file mode 100644 index 0000000000..07c940f47d --- /dev/null +++ b/salt/manager/tools/sbin_jinja/so-yara-update @@ -0,0 +1,41 @@ +#!/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +NOROOT=1 +. /usr/sbin/so-common + +echo "Starting to check for yara rule updates at $(date)..." + +newcounter=0 +excludedcounter=0 +excluded_rules=({{ EXCLUDEDRULES | join(' ') }}) + +# Pull down the SO Rules +SORULEDIR=/nsm/rules/yara +OUTPUTDIR=/opt/so/saltstack/local/salt/strelka/rules + +mkdir -p $OUTPUTDIR +# remove all rules prior to copy so we can clear out old rules +rm -f $OUTPUTDIR/* + +for i in $(find $SORULEDIR -name "*.yar" -o -name "*.yara"); do + rule_name=$(echo $i | awk -F '/' '{print $NF}') + if [[ ! "${excluded_rules[*]}" =~ ${rule_name} ]]; then + echo "Adding rule: $rule_name..." + cp $i $OUTPUTDIR/$rule_name + ((newcounter++)) + else + echo "Excluding rule: $rule_name..." + ((excludedcounter++)) + fi +done + +if [ "$newcounter" -gt 0 ] || [ "$excludedcounter" -gt 0 ];then + echo "$newcounter rules added." + echo "$excludedcounter rule(s) excluded." +fi + +echo "Finished rule updates at $(date)..." From e8ae60901233b06742a915cb64940e8564d75f37 Mon Sep 17 00:00:00 2001 From: Wes Date: Fri, 8 Mar 2024 16:27:17 +0000 Subject: [PATCH 118/140] Add Strelka rules watch back --- salt/strelka/backend/enabled.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/strelka/backend/enabled.sls b/salt/strelka/backend/enabled.sls index 9ebb1a1488..fc56f41971 100644 --- a/salt/strelka/backend/enabled.sls +++ b/salt/strelka/backend/enabled.sls @@ -42,8 +42,8 @@ strelka_backend: {% endfor %} {% endif %} - restart_policy: on-failure - #- watch: - # - file: strelkarules + - watch: + - file: strelkarules delete_so-strelka-backend_so-status.disabled: file.uncomment: From 6680e023e4f9dcec593d78967c9f6a49eba63856 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 8 Mar 2024 12:16:59 -0500 Subject: [PATCH 119/140] Update soc_pcap.yaml --- salt/pcap/soc_pcap.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/salt/pcap/soc_pcap.yaml b/salt/pcap/soc_pcap.yaml index 32204a23a4..96bc3831da 100644 --- a/salt/pcap/soc_pcap.yaml +++ b/salt/pcap/soc_pcap.yaml @@ -4,32 +4,32 @@ pcap: helpLink: stenographer.html config: maxdirectoryfiles: - description: The maximum number of packet/index files to create before deleting old files. + description: By default, Stenographer limits the number of files in the pcap directory to 30000 to avoid limitations with the ext3 filesystem. However, if you're using the ext4 or xfs filesystems, then it is safe to increase this value. So if you have a large amount of storage and find that you only have 3 weeks worth of PCAP on disk while still having plenty of free space, then you may want to increase this default setting. helpLink: stenographer.html diskfreepercentage: - description: The disk space percent to always keep free for PCAP + description: Stenographer will purge old PCAP on a regular basis to keep the disk free percentage at this level. If you have a distributed deployment with dedicated forward nodes, then the default value of 10 should be reasonable since Stenographer should be the main consumer of disk space in the /nsm partition. However, if you have systems that run both Stenographer and :ref:`elasticsearch` at the same time (like eval and standalone installations), then you’ll want to make sure that this value is no lower than 21 so that you avoid Elasticsearch hitting its watermark setting at 80% disk usage. If you have an older standalone installation, then you may need to manually change this value to 21. helpLink: stenographer.html blocks: - description: The number of 1MB packet blocks used by AF_PACKET to store packets in memory, per thread. You shouldn't need to change this. + description: The number of 1MB packet blocks used by Stenographer and AF_PACKET to store packets in memory, per thread. You shouldn't need to change this. advanced: True helpLink: stenographer.html preallocate_file_mb: - description: File size to pre-allocate for individual PCAP files. You shouldn't need to change this. + description: File size to pre-allocate for individual Stenographer PCAP files. You shouldn't need to change this. advanced: True helpLink: stenographer.html aiops: - description: The max number of async writes to allow at once. + description: The max number of async writes to allow for Stenographer at once. advanced: True helpLink: stenographer.html pin_to_cpu: - description: Enable CPU pinning for PCAP. + description: Enable CPU pinning for Stenographer PCAP. advanced: True helpLink: stenographer.html cpus_to_pin_to: - description: CPU to pin PCAP to. Currently only a single CPU is supported. + description: CPU to pin Stenographer PCAP to. Currently only a single CPU is supported. advanced: True helpLink: stenographer.html disks: - description: List of disks to use for PCAP. This is currently not used. + description: List of disks to use for Stenographer PCAP. This is currently not used. advanced: True helpLink: stenographer.html From 6f05c3976b45337fae4699c953345a80179b398b Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Fri, 8 Mar 2024 11:29:46 -0700 Subject: [PATCH 120/140] Updated RulesRepo for New Strelka Structure --- salt/soc/defaults.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 5699c77222..7be2db7725 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1132,7 +1132,8 @@ soc: compileYaraPythonScriptPath: /opt/so/conf/strelka/compile_yara.py reposFolder: /opt/sensoroni/yara/repos rulesRepos: - - https://github.com/Security-Onion-Solutions/securityonion-yara + - repo: https://github.com/Security-Onion-Solutions/securityonion-yara + license: DRL yaraRulesFolder: /opt/sensoroni/yara/rules suricataengine: communityRulesFile: /nsm/rules/suricata/emerging-all.rules From a55e04e64a0fe67ec20b2813df0aec7fa679ad1b Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 8 Mar 2024 15:48:53 -0500 Subject: [PATCH 121/140] pcap improvements --- salt/pcap/config.sls | 7 ------- salt/pcap/init.sls | 9 +++++++++ salt/sensoroni/enabled.sls | 3 --- salt/sensoroni/files/sensoroni.json | 12 +----------- salt/suricata/enabled.sls | 5 ----- salt/suricata/init.sls | 1 + salt/suricata/pcap.sls | 10 ++++++++-- 7 files changed, 19 insertions(+), 28 deletions(-) diff --git a/salt/pcap/config.sls b/salt/pcap/config.sls index 9ea5cee658..eb37765c5c 100644 --- a/salt/pcap/config.sls +++ b/salt/pcap/config.sls @@ -72,13 +72,6 @@ stenoca: - user: 941 - group: 939 -pcapdir: - file.directory: - - name: /nsm/pcap - - user: 941 - - group: 941 - - makedirs: True - pcaptmpdir: file.directory: - name: /nsm/pcaptmp diff --git a/salt/pcap/init.sls b/salt/pcap/init.sls index 9de272ad7a..7a172e8fd8 100644 --- a/salt/pcap/init.sls +++ b/salt/pcap/init.sls @@ -15,3 +15,12 @@ include: {% else %} - pcap.disabled {% endif %} + +# This directory needs to exist regardless of whether STENO is enabled or not, in order for +# Sensoroni to be able to look at old steno PCAP data +pcapdir: + file.directory: + - name: /nsm/pcap + - user: 941 + - group: 941 + - makedirs: True \ No newline at end of file diff --git a/salt/sensoroni/enabled.sls b/salt/sensoroni/enabled.sls index 6dc3df2bd4..3f05568a03 100644 --- a/salt/sensoroni/enabled.sls +++ b/salt/sensoroni/enabled.sls @@ -23,10 +23,7 @@ so-sensoroni: - /opt/so/conf/sensoroni/sensoroni.json:/opt/sensoroni/sensoroni.json:ro - /opt/so/conf/sensoroni/analyzers:/opt/sensoroni/analyzers:rw - /opt/so/log/sensoroni:/opt/sensoroni/logs:rw - {% if GLOBALS.pcap_engine == "SURICATA" %} - /nsm/suripcap/:/nsm/suripcap:rw - - /nsm/suripcaptmp:/nsm/suripcaptmp:rw - {% endif %} {% if DOCKER.containers['so-sensoroni'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-sensoroni'].custom_bind_mounts %} - {{ BIND }} diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index eb9c1131d8..f40f73167f 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -23,29 +23,19 @@ "statickeyauth": { "apiKey": "{{ GLOBALS.sensoroni_key }}" {% if GLOBALS.is_sensor %} -{% from 'pcap/config.map.jinja' import PCAPMERGED %} -{% from 'suricata/map.jinja' import SURICATAMERGED %} -{# if PCAPMERGED.enabled is true then we know that steno is the pcap engine #} -{# if it is false, then user has steno disabled in ui or has selected suricata for pcap engine #} -{%- if PCAPMERGED.enabled %} }, "stenoquery": { "executablePath": "/opt/sensoroni/scripts/stenoquery.sh", "pcapInputPath": "/nsm/pcap", "pcapOutputPath": "/nsm/pcapout" - } -{%- elif GLOBALS.pcap_engine == "SURICATA" and SURICATAMERGED.enabled %} }, "suriquery": { "executablePath": "/opt/sensoroni/scripts/suriquery.sh", "pcapInputPath": "/nsm/suripcap", "pcapOutputPath": "/nsm/pcapout", "pcapMaxCount": {{ SENSORONIMERGED.config.suripcap.pcapMaxCount }} - } -{% endif %} -{%- else %} - } {%- endif %} + } } } } diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index fed5783e1d..d351605276 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -12,9 +12,6 @@ include: - suricata.config - suricata.sostatus -{% if GLOBALS.pcap_engine == "SURICATA" %} - - suricata.pcap -{% endif %} so-suricata: docker_container.running: @@ -36,9 +33,7 @@ so-suricata: - /nsm/suricata/:/nsm/:rw - /nsm/suricata/extracted:/var/log/suricata//filestore:rw - /opt/so/conf/suricata/bpf:/etc/suricata/bpf:ro - {% if GLOBALS.pcap_engine == "SURICATA" %} - /nsm/suripcap/:/nsm/suripcap:rw - {% endif %} {% if DOCKER.containers['so-suricata'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-suricata'].custom_bind_mounts %} - {{ BIND }} diff --git a/salt/suricata/init.sls b/salt/suricata/init.sls index 64a000109d..f6ddce862b 100644 --- a/salt/suricata/init.sls +++ b/salt/suricata/init.sls @@ -7,6 +7,7 @@ {% from 'suricata/map.jinja' import SURICATAMERGED %} include: + - suricata.pcap {% if SURICATAMERGED.enabled and GLOBALS.role != 'so-import' %} - suricata.enabled {% elif GLOBALS.role == 'so-import' %} diff --git a/salt/suricata/pcap.sls b/salt/suricata/pcap.sls index a3cbafa0ac..665262477d 100644 --- a/salt/suricata/pcap.sls +++ b/salt/suricata/pcap.sls @@ -1,14 +1,18 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'suricata/map.jinja' import SURICATAMERGED %} +# This directory needs to exist regardless of whether SURIPCAP is enabled or not, in order for +# Sensoroni to be able to look at old Suricata PCAP data suripcapdir: file.directory: - name: /nsm/suripcap - user: 940 - group: 939 - - mode: 755 + - mode: 775 - makedirs: True +{% if GLOBALS.pcap_engine == "SURICATA" %} + {# there should only be 1 interface in af-packet so we can just reference the first list item #} {% for i in range(1, SURICATAMERGED.config['af-packet'][0].threads + 1) %} @@ -17,6 +21,8 @@ suripcapthread{{i}}dir: - name: /nsm/suripcap/{{i}} - user: 940 - group: 939 - - mode: 755 + - mode: 775 {% endfor %} + +{% endif %} From a892352b612627c05ae83da48eb0bbc383c27d9c Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 8 Mar 2024 16:43:29 -0500 Subject: [PATCH 122/140] Update soc_pcap.yaml --- salt/pcap/soc_pcap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/pcap/soc_pcap.yaml b/salt/pcap/soc_pcap.yaml index 96bc3831da..65fb99d869 100644 --- a/salt/pcap/soc_pcap.yaml +++ b/salt/pcap/soc_pcap.yaml @@ -7,7 +7,7 @@ pcap: description: By default, Stenographer limits the number of files in the pcap directory to 30000 to avoid limitations with the ext3 filesystem. However, if you're using the ext4 or xfs filesystems, then it is safe to increase this value. So if you have a large amount of storage and find that you only have 3 weeks worth of PCAP on disk while still having plenty of free space, then you may want to increase this default setting. helpLink: stenographer.html diskfreepercentage: - description: Stenographer will purge old PCAP on a regular basis to keep the disk free percentage at this level. If you have a distributed deployment with dedicated forward nodes, then the default value of 10 should be reasonable since Stenographer should be the main consumer of disk space in the /nsm partition. However, if you have systems that run both Stenographer and :ref:`elasticsearch` at the same time (like eval and standalone installations), then you’ll want to make sure that this value is no lower than 21 so that you avoid Elasticsearch hitting its watermark setting at 80% disk usage. If you have an older standalone installation, then you may need to manually change this value to 21. + description: Stenographer will purge old PCAP on a regular basis to keep the disk free percentage at this level. If you have a distributed deployment with dedicated forward nodes, then the default value of 10 should be reasonable since Stenographer should be the main consumer of disk space in the /nsm partition. However, if you have systems that run both Stenographer and Elasticsearch at the same time (like eval and standalone installations), then you’ll want to make sure that this value is no lower than 21 so that you avoid Elasticsearch hitting its watermark setting at 80% disk usage. If you have an older standalone installation, then you may need to manually change this value to 21. helpLink: stenographer.html blocks: description: The number of 1MB packet blocks used by Stenographer and AF_PACKET to store packets in memory, per thread. You shouldn't need to change this. From 34d5954e169972e21e412fb236f8bfc80cca788e Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 11 Mar 2024 09:12:05 -0400 Subject: [PATCH 123/140] Fix indent --- salt/manager/init.sls | 86 +++++++++++++++++++++---------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/salt/manager/init.sls b/salt/manager/init.sls index ee564dce86..c62a419998 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -118,49 +118,49 @@ rules_dir: - makedirs: True {% if STRELKAMERGED.rules.enabled %} - strelkarepos: - file.managed: - - name: /opt/so/conf/strelka/repos.txt - - source: salt://strelka/rules/repos.txt.jinja - - template: jinja - - defaults: - STRELKAREPOS: {{ STRELKAMERGED.rules.repos }} - - makedirs: True - strelka-yara-update: - {% if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} - cron.present: - {% else %} - cron.absent: - {% endif %} - - user: socore - - name: '/usr/sbin/so-yara-update >> /opt/so/log/yarasync/yara-update.log 2>&1' - - identifier: strelka-yara-update - - hour: '7' - - minute: '1' - strelka-yara-download: - {% if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} - cron.present: - {% else %} - cron.absent: - {% endif %} - - user: socore - - name: '/usr/sbin/so-yara-download >> /opt/so/log/yarasync/yara-download.log 2>&1' - - identifier: strelka-yara-download - - hour: '7' - - minute: '1' - {% if not GLOBALS.airgap %} - update_yara_rules: - cmd.run: - - name: /usr/sbin/so-yara-update - - onchanges: - - file: yara_update_scripts - download_yara_rules: - cmd.run: - - name: /usr/sbin/so-yara-download - - onchanges: - - file: yara_update_scripts - {% endif %} - {% endif %} +strelkarepos: + file.managed: + - name: /opt/so/conf/strelka/repos.txt + - source: salt://strelka/rules/repos.txt.jinja + - template: jinja + - defaults: + STRELKAREPOS: {{ STRELKAMERGED.rules.repos }} + - makedirs: True +strelka-yara-update: + {% if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} + cron.present: + {% else %} + cron.absent: + {% endif %} + - user: socore + - name: '/usr/sbin/so-yara-update >> /opt/so/log/yarasync/yara-update.log 2>&1' + - identifier: strelka-yara-update + - hour: '7' + - minute: '1' +strelka-yara-download: + {% if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} + cron.present: + {% else %} + cron.absent: + {% endif %} + - user: socore + - name: '/usr/sbin/so-yara-download >> /opt/so/log/yarasync/yara-download.log 2>&1' + - identifier: strelka-yara-download + - hour: '7' + - minute: '1' +{% if not GLOBALS.airgap %} +update_yara_rules: + cmd.run: + - name: /usr/sbin/so-yara-update + - onchanges: + - file: yara_update_scripts +download_yara_rules: + cmd.run: + - name: /usr/sbin/so-yara-download + - onchanges: + - file: yara_update_scripts +{% endif %} +{% endif %} {% else %} From a8403c63c73d37ad97ee0e5565fe8c3109c4019c Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 11 Mar 2024 09:35:54 -0400 Subject: [PATCH 124/140] Create local salt dir for stig Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/manager/tools/sbin/soup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 8b5d197515..0289310123 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -537,6 +537,8 @@ up_to_2.4.40() { up_to_2.4.50() { echo "Creating additional pillars.." mkdir -p /opt/so/saltstack/local/pillar/stig/ + mkdir -p /opt/so/saltstack/local/salt/stig/ + chown socore:socore /opt/so/saltstack/local/salt/stig/ touch /opt/so/saltstack/local/pillar/stig/adv_stig.sls touch /opt/so/saltstack/local/pillar/stig/soc_stig.sls From 907cf9f9924d25c3e309a32900c9a522ffa8a212 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 11 Mar 2024 12:20:28 -0400 Subject: [PATCH 125/140] transition pcap --- salt/bpf/pcap.map.jinja | 17 ++++++++++------- salt/global/soc_global.yaml | 6 +++--- salt/suricata/map.jinja | 2 +- salt/suricata/pcap.sls | 2 +- salt/telegraf/scripts/oldpcap.sh | 2 +- 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/salt/bpf/pcap.map.jinja b/salt/bpf/pcap.map.jinja index c1d7562ccd..a6deae4f41 100644 --- a/salt/bpf/pcap.map.jinja +++ b/salt/bpf/pcap.map.jinja @@ -1,7 +1,10 @@ -{% import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %} -{% set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %} -{% import 'bpf/macros.jinja' as MACROS %} - -{{ MACROS.remove_comments(BPFMERGED, 'pcap') }} - -{% set PCAPBPF = BPFMERGED.pcap %} +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% if GLOBALS.pcap_engine == "TRANSITION" %} +{% set PCAPBPF = "ip and host 255.255.255.1 and port 1" %} +{% else %} +{% import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %} +{% set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %} +{% import 'bpf/macros.jinja' as MACROS %} +{{ MACROS.remove_comments(BPFMERGED, 'pcap') }} +{% set PCAPBPF = BPFMERGED.pcap %} +{% endif %} diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml index d707fb1ccc..a48476214f 100644 --- a/salt/global/soc_global.yaml +++ b/salt/global/soc_global.yaml @@ -15,9 +15,9 @@ global: regexFailureMessage: You must enter either ZEEK or SURICATA. global: True pcapengine: - description: Which engine to use for generating pcap. Options are STENO and SURICATA. - regex: ^(STENO|SURICATA)$ - regexFailureMessage: You must enter either STENO or SURICATA. + description: Which engine to use for generating pcap. Options are STENO, SURICATA or TRANSITION. + regex: ^(STENO|SURICATA|TRANSITION)$ + regexFailureMessage: You must enter either STENO, SURICATA or TRANSITION. global: True ids: description: Which IDS engine to use. Currently only Suricata is supported. diff --git a/salt/suricata/map.jinja b/salt/suricata/map.jinja index 6ba3c3b734..7f7b04aeff 100644 --- a/salt/suricata/map.jinja +++ b/salt/suricata/map.jinja @@ -9,7 +9,7 @@ {% set surimeta_filestore_index = [] %} {# before we change outputs back to list, enable pcap-log if suricata is the pcapengine #} -{% if GLOBALS.pcap_engine == "SURICATA" %} +{% if GLOBALS.pcap_engine in ["SURICATA", "TRANSITION"] %} {% do SURICATAMERGED.config.outputs['pcap-log'].update({'enabled': 'yes'}) %} {# move the items in suricata.pcap into suricata.config.outputs.pcap-log. these items were placed under suricata.config for ease of access in SOC #} {% do SURICATAMERGED.config.outputs['pcap-log'].update({'compression': SURICATAMERGED.pcap.compression}) %} diff --git a/salt/suricata/pcap.sls b/salt/suricata/pcap.sls index 665262477d..87b568f964 100644 --- a/salt/suricata/pcap.sls +++ b/salt/suricata/pcap.sls @@ -11,7 +11,7 @@ suripcapdir: - mode: 775 - makedirs: True -{% if GLOBALS.pcap_engine == "SURICATA" %} +{% if GLOBALS.pcap_engine in ["SURICATA", "TRANSITION"] %} {# there should only be 1 interface in af-packet so we can just reference the first list item #} {% for i in range(1, SURICATAMERGED.config['af-packet'][0].threads + 1) %} diff --git a/salt/telegraf/scripts/oldpcap.sh b/salt/telegraf/scripts/oldpcap.sh index 438ce912c6..876ff7835a 100644 --- a/salt/telegraf/scripts/oldpcap.sh +++ b/salt/telegraf/scripts/oldpcap.sh @@ -5,7 +5,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- if GLOBALS.pcap_engine == "SURICATA" %} +{%- if GLOBALS.pcap_engine in ["SURICATA", "TRANSITION"] %} PCAPLOC=/host/nsm/suripcap {%- else %} PCAPLOC=/host/nsm/pcap From b5d8df7fb2ab72420d4fd35efe71cd9f3ead586b Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 11 Mar 2024 13:45:57 -0400 Subject: [PATCH 126/140] auto-convert email addresses to lowercase during setup --- setup/so-whiptail | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index ede138d26b..5e2a2de0e9 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -195,10 +195,12 @@ whiptail_create_web_user() { [ -n "$TESTING" ] && return WEBUSER=$(whiptail --title "$whiptail_title" --inputbox \ - "Please enter an email address to create an administrator account for the Security Onion Console (SOC) web interface.\n\nThis will also be used for Elasticsearch and Kibana." 12 60 "$1" 3>&1 1>&2 2>&3) + "Please enter an email address to create an administrator account for the Security Onion Console (SOC) web interface.\n\nThis will also be used for Elasticsearch and Kibana.\n\nMust only include letters, numbers, or + - _ % . @ characters. All capitalized letters will be converted to lowercase." 12 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus + + WEBUSER=${WEBUSER,,} } whiptail_create_web_user_password1() { From cd28c00d67dd658bf247e8d3c1401f35c67f7e80 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 11 Mar 2024 13:47:31 -0400 Subject: [PATCH 127/140] auto-convert email addresses to lowercase during setup --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 5e2a2de0e9..ff8c9fe8d5 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -195,7 +195,7 @@ whiptail_create_web_user() { [ -n "$TESTING" ] && return WEBUSER=$(whiptail --title "$whiptail_title" --inputbox \ - "Please enter an email address to create an administrator account for the Security Onion Console (SOC) web interface.\n\nThis will also be used for Elasticsearch and Kibana.\n\nMust only include letters, numbers, or + - _ % . @ characters. All capitalized letters will be converted to lowercase." 12 60 "$1" 3>&1 1>&2 2>&3) + "Please enter an email address to create an administrator account for the Security Onion Console (SOC) web interface.\n\nThis will also be used for Elasticsearch and Kibana.\n\nMust only include letters, numbers, or + - _ % . @ characters. All capitalized letters will be converted to lowercase." 15 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus From ba32b3e6e9d23a7c34fadef272f5bf8ec2e52ae3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 11 Mar 2024 14:07:45 -0400 Subject: [PATCH 128/140] fix bpf for transition --- salt/bpf/pcap.map.jinja | 2 +- salt/soc/defaults.yaml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/bpf/pcap.map.jinja b/salt/bpf/pcap.map.jinja index a6deae4f41..4d8fef4609 100644 --- a/salt/bpf/pcap.map.jinja +++ b/salt/bpf/pcap.map.jinja @@ -1,6 +1,6 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% if GLOBALS.pcap_engine == "TRANSITION" %} -{% set PCAPBPF = "ip and host 255.255.255.1 and port 1" %} +{% set PCAPBPF = ["ip and host 255.255.255.1 and port 1"] %} {% else %} {% import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %} {% set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %} diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 5699c77222..7be2db7725 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1132,7 +1132,8 @@ soc: compileYaraPythonScriptPath: /opt/so/conf/strelka/compile_yara.py reposFolder: /opt/sensoroni/yara/repos rulesRepos: - - https://github.com/Security-Onion-Solutions/securityonion-yara + - repo: https://github.com/Security-Onion-Solutions/securityonion-yara + license: DRL yaraRulesFolder: /opt/sensoroni/yara/rules suricataengine: communityRulesFile: /nsm/rules/suricata/emerging-all.rules From 61a183b7fc567ebdce5f9252a6af87d0330fda19 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Mon, 11 Mar 2024 15:55:39 -0400 Subject: [PATCH 129/140] Add regex defaults --- salt/soc/defaults.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 7be2db7725..197aee0709 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1078,8 +1078,10 @@ soc: kratos: hostUrl: elastalertengine: + allowRegex: '' autoUpdateEnabled: false - communityRulesImportFrequencySeconds: 180 + communityRulesImportFrequencySeconds: 86400 + denyRegex: '.*' elastAlertRulesFolder: /opt/sensoroni/elastalert rulesFingerprintFile: /opt/sensoroni/fingerprints/sigma.fingerprint sigmaRulePackages: @@ -1128,15 +1130,19 @@ soc: userFiles: - rbac/users_roles strelkaengine: + allowRegex: '' autoUpdateEnabled: false compileYaraPythonScriptPath: /opt/so/conf/strelka/compile_yara.py + denyRegex: '.*' reposFolder: /opt/sensoroni/yara/repos rulesRepos: - repo: https://github.com/Security-Onion-Solutions/securityonion-yara license: DRL yaraRulesFolder: /opt/sensoroni/yara/rules suricataengine: + allowRegex: '' communityRulesFile: /nsm/rules/suricata/emerging-all.rules + denyRegex: '.*' rulesFingerprintFile: /opt/sensoroni/fingerprints/emerging-all.fingerprint client: enableReverseLookup: false From 72acb11925bd85afcbd1fc59f167ff96520fb253 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 11 Mar 2024 19:04:51 -0400 Subject: [PATCH 130/140] Update soc_suricata.yaml --- salt/suricata/soc_suricata.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index da7586e97b..c61c041235 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -59,8 +59,8 @@ suricata: regexFailureMessage: You must enter either yes or no. helpLink: suricata.html conditional: - description: Set to "all" to capture PCAP for all flows. Set to "alert" to capture PCAP just for alerts or set to "tag" to capture PCAP for just tagged rules. - regex: ^(all|alert|tag)$ + description: Set to "all" to capture PCAP for all flows. Set to "alerts" to capture PCAP just for alerts or set to "tag" to capture PCAP for just tagged rules. + regex: ^(all|alerts|tag)$ regexFailureMessage: You must enter either all, alert or tag. helpLink: suricata.html dir: From 3e0fb3f8bb953f32dde333b25ad35266689e6fbc Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 12 Mar 2024 10:18:27 -0400 Subject: [PATCH 131/140] Update so-saltstack-update --- salt/manager/tools/sbin/so-saltstack-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/so-saltstack-update b/salt/manager/tools/sbin/so-saltstack-update index b15fce0089..4be8f095cf 100755 --- a/salt/manager/tools/sbin/so-saltstack-update +++ b/salt/manager/tools/sbin/so-saltstack-update @@ -47,7 +47,7 @@ got_root(){ got_root if [ $# -ne 1 ] ; then - BRANCH=master + BRANCH=2.4/main else BRANCH=$1 fi From 06013e2c6fd75d04cab6fda7ccc10c455e2c36f5 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Wed, 13 Mar 2024 07:23:43 -0400 Subject: [PATCH 132/140] Gen packages post-SOUP --- salt/manager/tools/sbin/soup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 0289310123..a585f877c7 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -430,7 +430,8 @@ post_to_2.4.50() { } post_to_2.4.60() { - echo "Nothing to apply" + echo "Regenerating Elastic Agent Installers..." + so-elastic-agent-gen-installers POSTVERSION=2.4.60 } From 1a829190ac4ba8777ac68bc865bfc6944be55d92 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 13 Mar 2024 09:46:44 -0400 Subject: [PATCH 133/140] remove modules if detections disabled --- salt/soc/defaults.yaml | 2 +- salt/soc/merged.map.jinja | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 197aee0709..de372a98fd 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1081,7 +1081,7 @@ soc: allowRegex: '' autoUpdateEnabled: false communityRulesImportFrequencySeconds: 86400 - denyRegex: '.*' + denyRegex: '' elastAlertRulesFolder: /opt/sensoroni/elastalert rulesFingerprintFile: /opt/sensoroni/fingerprints/sigma.fingerprint sigmaRulePackages: diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index bc7c5cada7..2012917af2 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -30,6 +30,13 @@ {# since cases is not a valid soc config item and only used for the map files, remove it from being placed in the config #} {% do SOCMERGED.config.server.modules.pop('cases') %} +{# remove these modules if detections is disabled #} +{% if not SOCMERGED.config.server.client.detectionsEnabled %} +{% do SOCMERGED.config.server.modules.pop('elastalertengine') %} +{% do SOCMERGED.config.server.modules.pop('strelkaengine') %} +{% do SOCMERGED.config.server.modules.pop('suricataengine') %} +{% endif %} + {% if pillar.manager.playbook == 0 %} {% do SOCMERGED.config.server.client.inactiveTools.append('toolPlaybook') %} {% endif %} From b9702d02db13c83765894067c9b07732ee41eff7 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 13 Mar 2024 11:24:26 -0400 Subject: [PATCH 134/140] Update init.sls --- salt/salt/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/salt/init.sls b/salt/salt/init.sls index a190a84eb2..b2ea31a65a 100644 --- a/salt/salt/init.sls +++ b/salt/salt/init.sls @@ -10,3 +10,4 @@ salt_bootstrap: - name: /usr/sbin/bootstrap-salt.sh - source: salt://salt/scripts/bootstrap-salt.sh - mode: 755 + - show_changes: False From 275a678fa1a90369d2d64ff2260679975384f548 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 13 Mar 2024 13:49:44 -0400 Subject: [PATCH 135/140] removed unused property --- salt/sensoroni/files/sensoroni.json | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index f40f73167f..547e52ada1 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -30,7 +30,6 @@ "pcapOutputPath": "/nsm/pcapout" }, "suriquery": { - "executablePath": "/opt/sensoroni/scripts/suriquery.sh", "pcapInputPath": "/nsm/suripcap", "pcapOutputPath": "/nsm/pcapout", "pcapMaxCount": {{ SENSORONIMERGED.config.suripcap.pcapMaxCount }} From 927fe9039d1dca052e96cbfcdd3db380fe49b672 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 13 Mar 2024 20:50:03 -0400 Subject: [PATCH 136/140] handle airgap when detections not enabled --- salt/soc/merged.map.jinja | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index 2012917af2..57abe7a489 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -35,16 +35,16 @@ {% do SOCMERGED.config.server.modules.pop('elastalertengine') %} {% do SOCMERGED.config.server.modules.pop('strelkaengine') %} {% do SOCMERGED.config.server.modules.pop('suricataengine') %} +{% elif pillar.global.airgap %} + {# if system is Airgap, don't autoupdate Yara & Sigma rules #} + {% do SOCMERGED.config.server.modules.elastalertengine.update({'autoUpdateEnabled': false}) %} + {% do SOCMERGED.config.server.modules.strelkaengine.update({'autoUpdateEnabled': false}) %} {% endif %} -{% if pillar.manager.playbook == 0 %} -{% do SOCMERGED.config.server.client.inactiveTools.append('toolPlaybook') %} {% endif %} -{# if system is Airgap, don't autoupdate Yara & Sigma rules #} -{% if pillar.global.airgap %} - {% do SOCMERGED.config.server.modules.elastalertengine.update({'autoUpdateEnabled': false}) %} - {% do SOCMERGED.config.server.modules.strelkaengine.update({'autoUpdateEnabled': false}) %} +{% if pillar.manager.playbook == 0 %} +{% do SOCMERGED.config.server.client.inactiveTools.append('toolPlaybook') %} {% endif %} {% set standard_actions = SOCMERGED.config.pop('actions') %} From 844cfe55cd0ea40317a51f9cee33a801d690d647 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 13 Mar 2024 20:52:17 -0400 Subject: [PATCH 137/140] handle airgap when detections not enabled --- salt/soc/merged.map.jinja | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index 57abe7a489..c22ed2210d 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -41,8 +41,6 @@ {% do SOCMERGED.config.server.modules.strelkaengine.update({'autoUpdateEnabled': false}) %} {% endif %} -{% endif %} - {% if pillar.manager.playbook == 0 %} {% do SOCMERGED.config.server.client.inactiveTools.append('toolPlaybook') %} {% endif %} From 284e0d84354a0d8b769a4582d0c68e52b749e669 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 14 Mar 2024 11:33:47 -0400 Subject: [PATCH 138/140] Update soc_suricata.yaml --- salt/suricata/soc_suricata.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index c61c041235..47e9e15030 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -28,19 +28,16 @@ suricata: description: Size in GB for total usage size of PCAP on disk. helplink: suricata.html compression: - description: Enable compression of Suricata PCAP. Currently unsupported + description: Enable compression of Suricata PCAP. advanced: True - readonly: True helpLink: suricata.html lz4-checksum: - description: Enable PCAP lz4 checksum. Currently unsupported + description: Enable PCAP lz4 checksum. advanced: True - readonly: True helpLink: suricata.html lz4-level: description: lz4 compression level of PCAP. 0 for no compression 16 for max compression. Currently unsupported advanced: True - readonly: True helpLink: suricata.html filename: description: Filename output for Suricata PCAP. From fd835f63947b152d3390c3b6f866c9f0e7c338a2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 14 Mar 2024 11:36:45 -0400 Subject: [PATCH 139/140] Update soc_suricata.yaml --- salt/suricata/soc_suricata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index 47e9e15030..7decaa6d32 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -36,7 +36,7 @@ suricata: advanced: True helpLink: suricata.html lz4-level: - description: lz4 compression level of PCAP. 0 for no compression 16 for max compression. Currently unsupported + description: lz4 compression level of PCAP. 0 for no compression 16 for max compression. advanced: True helpLink: suricata.html filename: From bb3bbd749c65ee45489b847a587e6be25a419d8a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 20 Mar 2024 10:20:04 -0400 Subject: [PATCH 140/140] 2.4.260 --- DOWNLOAD_AND_VERIFY_ISO.md | 22 ++++++++++----------- sigs/securityonion-2.4.60-20240320.iso.sig | Bin 0 -> 566 bytes 2 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 sigs/securityonion-2.4.60-20240320.iso.sig diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index a23d88d4dc..4493f210db 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -1,17 +1,17 @@ -### 2.4.50-20240220 ISO image released on 2024/02/20 +### 2.4.60-20240320 ISO image released on 2024/03/20 ### Download and Verify -2.4.50-20240220 ISO image: -https://download.securityonion.net/file/securityonion/securityonion-2.4.50-20240220.iso +2.4.60-20240320 ISO image: +https://download.securityonion.net/file/securityonion/securityonion-2.4.60-20240320.iso -MD5: BCA6476EF1BF79773D8EFB11700FDE8E -SHA1: 9FF0A304AA368BCD2EF2BE89AD47E65650241927 -SHA256: 49D7695EFFF6F3C4840079BF564F3191B585639816ADE98672A38017F25E9570 +MD5: 178DD42D06B2F32F3870E0C27219821E +SHA1: 73EDCD50817A7F6003FE405CF1808A30D034F89D +SHA256: DD334B8D7088A7B78160C253B680D645E25984BA5CCAB5CC5C327CA72137FC06 Signature for ISO image: -https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.50-20240220.iso.sig +https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.60-20240320.iso.sig Signing key: https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS @@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2. Download the signature file for the ISO: ``` -wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.50-20240220.iso.sig +wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.60-20240320.iso.sig ``` Download the ISO image: ``` -wget https://download.securityonion.net/file/securityonion/securityonion-2.4.50-20240220.iso +wget https://download.securityonion.net/file/securityonion/securityonion-2.4.60-20240320.iso ``` Verify the downloaded ISO image using the signature file: ``` -gpg --verify securityonion-2.4.50-20240220.iso.sig securityonion-2.4.50-20240220.iso +gpg --verify securityonion-2.4.60-20240320.iso.sig securityonion-2.4.60-20240320.iso ``` The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Fri 16 Feb 2024 11:36:25 AM EST using RSA key ID FE507013 +gpg: Signature made Tue 19 Mar 2024 03:17:58 PM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/sigs/securityonion-2.4.60-20240320.iso.sig b/sigs/securityonion-2.4.60-20240320.iso.sig new file mode 100644 index 0000000000000000000000000000000000000000..c0129ab64b049b783258081c8d3dde117b29aa95 GIT binary patch literal 566 zcmV-60?GY}0y6{v0SEvc79j-41gSkXz6^6dp_W8^5Ma0dP;e6k0%iH+<^T!_5PT3| zxBgIY6FsdE0HMAsL4Ya2;dUH@Tj-=rYUOi%4z*NE^@q+`;Erl=A8#u|b^A0~xO#6`Szzg5~-3VjT3Yys$sGkuxpcsZ@^e5$v7pXryod-7`cIChZTfeuV& zKIlAHEhH+#f`k%z&qebjUlS~w3XH&a0N!TFO3A(4TzcH@Bj#jfvyTbV)Eyc8J;5eF zL9BEEpso{FY9)Vhc3*2LAuZ&Xo~l7+Jb{y4Qr$U&^WQTRe+g57T7=d8`o!9)8dp2s zgkP&e^vc+xnu!=S9`UpxO0R#HROR;d$ZLxxt`ZVR0Ne_ywWWs$w3}i184t&>AnQ#OU-=HH6mAF^^|dFa z>sH%7KeQ%8Gt+!Vx#jt6YUO&Y^g8?yMOnxK9R2w4q_eENgMup&e%TRy7{8${`LFyP$=x?8XlQVruAYhoUiD`V~ehpF literal 0 HcmV?d00001