Merge pull request #13777 from Security-Onion-Solutions/2.4/dev

2.4.110

.github/DISCUSSION_TEMPLATE/2-4.yml

@@ -11,7 +11,6 @@ body:
       description: Which version of Security Onion 2.4.x are you asking about?
       options:
         -
-        - 2.4 Pre-release (Beta, Release Candidate)
         - 2.4.10
         - 2.4.20
         - 2.4.30
@@ -22,6 +21,7 @@ body:
         - 2.4.80
         - 2.4.90
         - 2.4.100
+        - 2.4.110
         - Other (please provide detail below)
     validations:
       required: true
@@ -32,9 +32,10 @@ body:
       options:
         -
         - Security Onion ISO image
-        - Network installation on Red Hat derivative like Oracle, Rocky, Alma, etc.
-        - Network installation on Ubuntu
-        - Network installation on Debian
+        - Cloud image (Amazon, Azure, Google)
+        - Network installation on Red Hat derivative like Oracle, Rocky, Alma, etc. (unsupported)
+        - Network installation on Ubuntu (unsupported)
+        - Network installation on Debian (unsupported)
         - Other (please provide detail below)
     validations:
       required: true

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,17 +1,17 @@
-### 2.4.100-20240903 ISO image released on 2024/09/03
+### 2.4.110-20241004 ISO image released on 2024/10/07
 
 
 ### Download and Verify
 
-2.4.100-20240903 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.4.100-20240903.iso
+2.4.110-20241004 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.4.110-20241004.iso
 
-MD5: 856BBB4F0764C0A479D8949725FC096B  
-SHA1: B3FCFB8F1031EB8AA833A90C6C5BB61328A73842  
-SHA256: 0103EB9D78970396BB47CBD18DA1FFE64524F5C1C559487A1B2D293E1882B265  
+MD5: 1641E4AFD65DB1C218BFAD22E33909C6  
+SHA1: 131E1115F7CA76302F72625CD80A212B91608114  
+SHA256: 8598EB03E52B332EF5445520445AD205C68A99BC030F8497F6EBDE1249B8B576  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.100-20240903.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.110-20241004.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS  
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.100-20240903.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.110-20241004.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.4.100-20240903.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.4.110-20241004.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.4.100-20240903.iso.sig securityonion-2.4.100-20240903.iso
+gpg --verify securityonion-2.4.110-20241004.iso.sig securityonion-2.4.110-20241004.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Sat 31 Aug 2024 05:05:05 PM EDT using RSA key ID FE507013
+gpg: Signature made Sat 05 Oct 2024 09:31:57 AM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <[email protected]>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

HOTFIX

@@ -1 +0,0 @@
-20240903

VERSION

@@ -1 +1 @@
-2.4.100
+2.4.110

pillar/top.sls

@@ -310,3 +310,5 @@ base:
   '*_desktop':
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
+    - stig.soc_stig
+    - soc.license

salt/allowed_states.map.jinja

@@ -202,7 +202,8 @@
       'so-desktop': [
           'ssl',
           'docker_clean',
-          'telegraf'
+          'telegraf',
+          'stig'
           ],
   }, grain='role') %}
 

salt/common/tools/sbin/so-common

@@ -8,12 +8,6 @@
 # Elastic agent is not managed by salt. Because of this we must store this base information in a 
 # script that accompanies the soup system. Since so-common is one of those special soup files, 
 # and since this same logic is required during installation, it's included in this file.
-ELASTIC_AGENT_TARBALL_VERSION="8.14.3"
-ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
-ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
-ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
-ELASTIC_AGENT_MD5="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
-ELASTIC_AGENT_EXPANSION_DIR=/nsm/elastic-fleet/artifacts/beats/elastic-agent
 
 DEFAULT_SALT_DIR=/opt/so/saltstack/default
 DOC_BASE_URL="https://docs.securityonion.net/en/2.4"
@@ -174,6 +168,46 @@ check_salt_minion_status() {
 	return $status
 }
 
+# Compare es versions and return the highest version
+compare_es_versions() {
+    # Save the original IFS
+    local OLD_IFS="$IFS"
+
+    IFS=.
+    local i ver1=($1) ver2=($2)
+
+    # Restore the original IFS
+    IFS="$OLD_IFS"
+
+    # Determine the maximum length between the two version arrays
+    local max_len=${#ver1[@]}
+    if [[ ${#ver2[@]} -gt $max_len ]]; then
+        max_len=${#ver2[@]}
+    fi
+
+    # Compare each segment of the versions
+    for ((i=0; i<max_len; i++)); do
+		# If a segment in ver1 or ver2 is missing, set it to 0
+        if [[ -z ${ver1[i]} ]]; then
+            ver1[i]=0
+        fi
+        if [[ -z ${ver2[i]} ]]; then
+            ver2[i]=0
+        fi
+        if ((10#${ver1[i]} > 10#${ver2[i]})); then
+            echo "$1"
+            return 0
+        fi
+        if ((10#${ver1[i]} < 10#${ver2[i]})); then
+            echo "$2"
+            return 0
+        fi
+    done
+
+    echo "$1"  # If versions are equal, return either
+    return 0
+}
+
 copy_new_files() {
   # Copy new files over to the salt dir
   cd $UPDATE_DIR
@@ -263,11 +297,6 @@ fail() {
 	exit 1
 }
 
-get_random_value() {
-	length=${1:-20}
-	head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1
-}
-
 get_agent_count() {
 	if [ -f /opt/so/log/agents/agentstatus.log ]; then
 		AGENTCOUNT=$(cat /opt/so/log/agents/agentstatus.log | grep -wF active | awk '{print $2}')
@@ -276,6 +305,27 @@ get_agent_count() {
 	fi
 }
 
+get_elastic_agent_vars() {
+	local path="${1:-/opt/so/saltstack/default}"
+	local defaultsfile="${path}/salt/elasticsearch/defaults.yaml"
+
+	if [ -f "$defaultsfile" ]; then
+		ELASTIC_AGENT_TARBALL_VERSION=$(egrep " +version: " $defaultsfile | awk -F: '{print $2}' | tr -d '[:space:]')
+		ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
+		ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
+		ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
+		ELASTIC_AGENT_MD5="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
+		ELASTIC_AGENT_EXPANSION_DIR=/nsm/elastic-fleet/artifacts/beats/elastic-agent
+	else
+		fail "Could not find salt/elasticsearch/defaults.yaml"
+	fi
+}
+
+get_random_value() {
+	length=${1:-20}
+	head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1
+}
+
 gpg_rpm_import() {
 	if [[ $is_oracle ]]; then
 	    if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then
@@ -627,6 +677,8 @@ has_uppercase() {
 }
 
 update_elastic_agent() {
+	local path="${1:-/opt/so/saltstack/default}"
+	get_elastic_agent_vars "$path"
 	echo "Checking if Elastic Agent update is necessary..."
 	download_and_verify "$ELASTIC_AGENT_URL" "$ELASTIC_AGENT_MD5_URL" "$ELASTIC_AGENT_FILE" "$ELASTIC_AGENT_MD5" "$ELASTIC_AGENT_EXPANSION_DIR"
 }

salt/common/tools/sbin/so-image-common

@@ -112,6 +112,10 @@ update_docker_containers() {
     container_list
   fi
 
+  # all the images using ELASTICSEARCHDEFAULTS.elasticsearch.version
+  # does not include so-elastic-fleet since that container uses so-elastic-agent image
+  local IMAGES_USING_ES_VERSION=("so-elasticsearch")
+
   rm -rf $SIGNPATH >> "$LOG_FILE" 2>&1 
   mkdir -p $SIGNPATH >> "$LOG_FILE" 2>&1 
 
@@ -139,15 +143,36 @@ update_docker_containers() {
       $PROGRESS_CALLBACK $i
     fi
 
+    if [[ " ${IMAGES_USING_ES_VERSION[*]} " =~ [[:space:]]${i}[[:space:]] ]]; then
+      # this is an es container so use version defined in elasticsearch defaults.yaml
+      local UPDATE_DIR='/tmp/sogh/securityonion'
+      if [ ! -d "$UPDATE_DIR" ]; then
+        UPDATE_DIR=/securityonion
+      fi
+      local v1=0
+      local v2=0
+      if [[ -f "$UPDATE_DIR/salt/elasticsearch/defaults.yaml" ]]; then
+        v1=$(egrep " +version: " "$UPDATE_DIR/salt/elasticsearch/defaults.yaml" | awk -F: '{print $2}' | tr -d '[:space:]')
+      fi
+      if [[ -f "$DEFAULT_SALT_DIR/salt/elasticsearch/defaults.yaml" ]]; then
+        v2=$(egrep " +version: " "$DEFAULT_SALT_DIR/salt/elasticsearch/defaults.yaml" | awk -F: '{print $2}' | tr -d '[:space:]')
+      fi
+      local highest_es_version=$(compare_es_versions "$v1" "$v2")
+      local image=$i:$highest_es_version$IMAGE_TAG_SUFFIX
+      local sig_url=https://sigs.securityonion.net/es-$highest_es_version/$image.sig
+    else
+      # this is not an es container so use the so version for the version
+      local image=$i:$VERSION$IMAGE_TAG_SUFFIX
+      local sig_url=https://sigs.securityonion.net/$VERSION/$image.sig
+    fi
     # Pull down the trusted docker image
-    local image=$i:$VERSION$IMAGE_TAG_SUFFIX
     run_check_net_err \
     "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" \
     "Could not pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1 
 
     # Get signature
     run_check_net_err \
-    "curl --retry 5 --retry-delay 60 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" \
+    "curl --retry 5 --retry-delay 60 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' $sig_url --output $SIGNPATH/$image.sig" \
     "Could not pull signature file for $image, please ensure connectivity to https://sigs.securityonion.net " \
     noretry >> "$LOG_FILE" 2>&1
     # Dump our hash values

salt/docker/init.sls

@@ -20,41 +20,41 @@ dockergroup:
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.6.33-1
-      - docker-ce: 5:26.1.4-1~debian.12~bookworm
-      - docker-ce-cli: 5:26.1.4-1~debian.12~bookworm
-      - docker-ce-rootless-extras: 5:26.1.4-1~debian.12~bookworm
+      - containerd.io: 1.7.21-1
+      - docker-ce: 5:27.2.0-1~debian.12~bookworm
+      - docker-ce-cli: 5:27.2.0-1~debian.12~bookworm
+      - docker-ce-rootless-extras: 5:27.2.0-1~debian.12~bookworm
     - hold: True
     - update_holds: True
 {%    elif grains.oscodename == 'jammy' %}
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.6.33-1
-      - docker-ce: 5:26.1.4-1~ubuntu.22.04~jammy
-      - docker-ce-cli: 5:26.1.4-1~ubuntu.22.04~jammy
-      - docker-ce-rootless-extras: 5:26.1.4-1~ubuntu.22.04~jammy
+      - containerd.io: 1.7.21-1
+      - docker-ce: 5:27.2.0-1~ubuntu.22.04~jammy
+      - docker-ce-cli: 5:27.2.0-1~ubuntu.22.04~jammy
+      - docker-ce-rootless-extras: 5:27.2.0-1~ubuntu.22.04~jammy
     - hold: True
     - update_holds: True
 {%    else %}
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.6.33-1
-      - docker-ce: 5:26.1.4-1~ubuntu.20.04~focal
-      - docker-ce-cli: 5:26.1.4-1~ubuntu.20.04~focal
-      - docker-ce-rootless-extras: 5:26.1.4-1~ubuntu.20.04~focal
+      - containerd.io: 1.7.21-1
+      - docker-ce: 5:27.2.0-1~ubuntu.20.04~focal
+      - docker-ce-cli: 5:27.2.0-1~ubuntu.20.04~focal
+      - docker-ce-rootless-extras: 5:27.2.0-1~ubuntu.20.04~focal
     - hold: True
     - update_holds: True
-{% endif %}
+{%   endif %}
 {% else %}
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.6.33-3.1.el9
-      - docker-ce: 3:26.1.4-1.el9
-      - docker-ce-cli: 1:26.1.4-1.el9
-      - docker-ce-rootless-extras: 26.1.4-1.el9
+      - containerd.io: 1.7.21-3.1.el9
+      - docker-ce: 3:27.2.0-1.el9
+      - docker-ce-cli: 1:27.2.0-1.el9
+      - docker-ce-rootless-extras: 27.2.0-1.el9
     - hold: True
     - update_holds: True
 {% endif %}

salt/elastalert/soc_elastalert.yaml

@@ -1,6 +1,6 @@
 elastalert:
   enabled:
-    description: You can enable or disable Elastalert.
+    description: Enables or disables the ElastAlert 2 process. This process is critical for ensuring alerts arrive in SOC, and for outbound notification delivery.
     helpLink: elastalert.html
   alerter_parameters:
     title: Custom Configuration Parameters

salt/elastic-fleet-package-registry/soc_elastic-fleet-package-registry.yaml

@@ -1,4 +1,4 @@
 elastic_fleet_package_registry:
   enabled:
-    description: You can enable or disable Elastic Fleet Package Registry.
+    description: Enables or disables the Fleet package registry process. This process must remain enabled to allow Elastic Agent packages to be updated.
     advanced: True

salt/elasticagent/enabled.sls

@@ -8,7 +8,6 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKER %}
 
-
 include:
   - elasticagent.config
   - elasticagent.sostatus

salt/elasticagent/soc_elasticagent.yaml

@@ -0,0 +1,4 @@
+elasticagent:
+  enabled:
+    description: Enables or disables the Elastic Agent process. This process must remain enabled to allow collection of node events.
+    advanced: True