diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index df8904e0a7..f815bb7cfd 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -1,17 +1,17 @@ -### 2.4.90-20240729 ISO image released on 2024/07/29 +### 2.4.100-20240829 ISO image released on 2024/08/29 ### Download and Verify -2.4.90-20240729 ISO image: -https://download.securityonion.net/file/securityonion/securityonion-2.4.90-20240729.iso +2.4.100-20240829 ISO image: +https://download.securityonion.net/file/securityonion/securityonion-2.4.100-20240829.iso -MD5: 9A7714F5922EE555F08675D25E6237D5 -SHA1: D3B331452627DB716906BA9F3922574DFA3852DC -SHA256: 5B0CE32543944DBC50C4E906857384211E1BE83EF409619778F18FC62017E0E0 +MD5: 377586C143FABD662DB414DEA49D46B7 +SHA1: 69D4B94522789AF47075A9FF1354B069679AC366 +SHA256: 52FBA5C8762B8DCF2945AD2837B3A19E63ADCC209AB510D7FD0F86AE713AA153 Signature for ISO image: -https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.90-20240729.iso.sig +https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.100-20240829.iso.sig Signing key: https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS @@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2. Download the signature file for the ISO: ``` -wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.90-20240729.iso.sig +wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.100-20240829.iso.sig ``` Download the ISO image: ``` -wget https://download.securityonion.net/file/securityonion/securityonion-2.4.90-20240729.iso +wget https://download.securityonion.net/file/securityonion/securityonion-2.4.100-20240829.iso ``` Verify the downloaded ISO image using the signature file: ``` -gpg --verify securityonion-2.4.90-20240729.iso.sig securityonion-2.4.90-20240729.iso +gpg --verify securityonion-2.4.100-20240829.iso.sig securityonion-2.4.100-20240829.iso ``` The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Thu 25 Jul 2024 06:51:11 PM EDT using RSA key ID FE507013 +gpg: Signature made Thu 29 Aug 2024 12:02:55 PM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/SECURITY.md b/SECURITY.md index 771d087a12..5001e4cd0e 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -5,9 +5,11 @@ | Version | Supported | | ------- | ------------------ | | 2.4.x | :white_check_mark: | -| 2.3.x | :white_check_mark: | +| 2.3.x | :x: | | 16.04.x | :x: | +Security Onion 2.3 has reached End Of Life and is no longer supported. + Security Onion 16.04 has reached End Of Life and is no longer supported. ## Reporting a Vulnerability diff --git a/VERSION b/VERSION index 38f621b256..fd912cb251 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.90 +2.4.100 diff --git a/salt/common/init.sls b/salt/common/init.sls index 51836daf60..8dd727f727 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -14,6 +14,11 @@ net.core.wmem_default: sysctl.present: - value: 26214400 +# Users are not a fan of console messages +kernel.printk: + sysctl.present: + - value: "3 4 1 3" + # Remove variables.txt from /tmp - This is temp rmvariablesfile: file.absent: diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 05c47a6c17..902aabaa3f 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -8,7 +8,7 @@ # Elastic agent is not managed by salt. Because of this we must store this base information in a # script that accompanies the soup system. Since so-common is one of those special soup files, # and since this same logic is required during installation, it's included in this file. -ELASTIC_AGENT_TARBALL_VERSION="8.10.4" +ELASTIC_AGENT_TARBALL_VERSION="8.14.3" ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5" ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 2a86b9f2ca..3007325873 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -95,6 +95,8 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|shutdown process" # server not yet ready (logstash waiting on elastic) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|contain valid certificates" # server not yet ready (logstash waiting on elastic) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failedaction" # server not yet ready (logstash waiting on elastic) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|block in start_workers" # server not yet ready (logstash waiting on elastic) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|block in buffer_initialize" # server not yet ready (logstash waiting on elastic) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no route to host" # server not yet ready EXCLUDED_ERRORS="$EXCLUDED_ERRORS|not running" # server not yet ready EXCLUDED_ERRORS="$EXCLUDED_ERRORS|unavailable" # server not yet ready @@ -147,6 +149,7 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status 200" # false positive (request successful, contained error string in content) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|app_layer.error" # false positive (suricata 7) in stats.log e.g. app_layer.error.imap.parser | Total | 0 EXCLUDED_ERRORS="$EXCLUDED_ERRORS|is not an ip string literal" # false positive (Open Canary logging out blank IP addresses) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncing rule" # false positive (rule sync log line includes rule name which can contain 'error') fi if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then @@ -170,6 +173,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|cannot join on an empty table" # InfluxDB flux query, import nodes EXCLUDED_ERRORS="$EXCLUDED_ERRORS|exhausting result iterator" # InfluxDB flux query mismatched table results (temporary data issue) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to finish run" # InfluxDB rare error, self-recoverable + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unable to gather disk name" # InfluxDB known error, can't read disks because the container doesn't have them mounted EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|use of closed" @@ -205,6 +209,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|detect-parse" # Suricata encountering a malformed rule EXCLUDED_ERRORS="$EXCLUDED_ERRORS|integrity check failed" # Detections: Exclude false positive due to automated testing EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncErrors" # Detections: Not an actual error + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Initialized license manager" # SOC log: before fields.status was changed to fields.licenseStatus fi RESULT=0 diff --git a/salt/common/tools/sbin_jinja/so-raid-status b/salt/common/tools/sbin_jinja/so-raid-status index 6cd8b84de0..3fe238c236 100755 --- a/salt/common/tools/sbin_jinja/so-raid-status +++ b/salt/common/tools/sbin_jinja/so-raid-status @@ -9,6 +9,9 @@ . /usr/sbin/so-common +software_raid=("SOSMN" "SOSMN-DE02" "SOSSNNV" "SOSSNNV-DE02" "SOS10k-DE02" "SOS10KNV" "SOS10KNV-DE02" "SOS10KNV-DE02" "SOS2000-DE02" "SOS-GOFAST-LT-DE02" "SOS-GOFAST-MD-DE02" "SOS-GOFAST-HV-DE02") +hardware_raid=("SOS1000" "SOS1000F" "SOSSN7200" "SOS5000" "SOS4000") + {%- if salt['grains.get']('sosmodel', '') %} {%- set model = salt['grains.get']('sosmodel') %} model={{ model }} @@ -16,33 +19,42 @@ model={{ model }} if [[ $model =~ ^(SO2AMI01|SO2AZI01|SO2GCI01)$ ]]; then exit 0 fi + +for i in "${software_raid[@]}"; do + if [[ "$model" == $i ]]; then + is_softwareraid=true + is_hwraid=false + break + fi +done + +for i in "${hardware_raid[@]}"; do + if [[ "$model" == $i ]]; then + is_softwareraid=false + is_hwraid=true + break + fi +done + {%- else %} echo "This is not an appliance" exit 0 {%- endif %} -if [[ $model =~ ^(SOS10K|SOS500|SOS1000|SOS1000F|SOS4000|SOSSN7200|SOSSNNV|SOSMN)$ ]]; then - is_bossraid=true -fi -if [[ $model =~ ^(SOSSNNV|SOSMN)$ ]]; then - is_swraid=true -fi -if [[ $model =~ ^(SOS10K|SOS500|SOS1000|SOS1000F|SOS4000|SOSSN7200)$ ]]; then - is_hwraid=true -fi check_nsm_raid() { PERCCLI=$(/opt/raidtools/perccli/perccli64 /c0/v0 show|grep RAID|grep Optl) MEGACTL=$(/opt/raidtools/megasasctl |grep optimal) - - if [[ $APPLIANCE == '1' ]]; then + if [[ "$model" == "SOS500" || "$model" == "SOS500-DE02" ]]; then + #This doesn't have raid + HWRAID=0 + else if [[ -n $PERCCLI ]]; then HWRAID=0 elif [[ -n $MEGACTL ]]; then HWRAID=0 else HWRAID=1 - fi - + fi fi } @@ -50,17 +62,27 @@ check_nsm_raid() { check_boss_raid() { MVCLI=$(/usr/local/bin/mvcli info -o vd |grep status |grep functional) MVTEST=$(/usr/local/bin/mvcli info -o vd | grep "No adapter") + BOSSNVMECLI=$(/usr/local/bin/mnv_cli info -o vd -i 0 | grep Functional) - # Check to see if this is a SM based system - if [[ -z $MVTEST ]]; then - if [[ -n $MVCLI ]]; then + # Is this NVMe Boss Raid? + if [[ "$model" =~ "-DE02" ]]; then + if [[ -n $BOSSNVMECLI ]]; then BOSSRAID=0 else BOSSRAID=1 fi else - # This doesn't have boss raid so lets make it 0 - BOSSRAID=0 + # Check to see if this is a SM based system + if [[ -z $MVTEST ]]; then + if [[ -n $MVCLI ]]; then + BOSSRAID=0 + else + BOSSRAID=1 + fi + else + # This doesn't have boss raid so lets make it 0 + BOSSRAID=0 + fi fi } @@ -79,14 +101,13 @@ SWRAID=0 BOSSRAID=0 HWRAID=0 -if [[ $is_hwraid ]]; then +if [[ "$is_hwraid" == "true" ]]; then check_nsm_raid + check_boss_raid fi -if [[ $is_bossraid ]]; then - check_boss_raid -fi -if [[ $is_swraid ]]; then +if [[ "$is_softwareraid" == "true" ]]; then check_software_raid + check_boss_raid fi sum=$(($SWRAID + $BOSSRAID + $HWRAID)) diff --git a/salt/elastalert/soc_elastalert.yaml b/salt/elastalert/soc_elastalert.yaml index 435c5be6a3..905fd38849 100644 --- a/salt/elastalert/soc_elastalert.yaml +++ b/salt/elastalert/soc_elastalert.yaml @@ -3,8 +3,8 @@ elastalert: description: You can enable or disable Elastalert. helpLink: elastalert.html alerter_parameters: - title: Alerter Parameters - description: Optional configuration parameters for additional alerters that can be enabled for all Sigma rules. Filter for 'Alerter' in this Configuration screen to find the setting that allows these alerters to be enabled within the SOC ElastAlert module. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. + title: Custom Configuration Parameters + description: Optional configuration parameters made available as defaults for all rules and alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available configuration parameters. Requires a valid Security Onion license key. global: True multiline: True syntax: yaml diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 2d9ab97a14..48b24809e6 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -97,6 +97,7 @@ elasticfleet: - symantec_endpoint - system - tcp + - tenable_io - tenable_sc - ti_abusech - ti_anomali diff --git a/salt/elasticfleet/files/integrations-dynamic/fleet-server/fleet-server.json b/salt/elasticfleet/files/integrations-dynamic/fleet-server/fleet-server.json new file mode 100644 index 0000000000..116f785009 --- /dev/null +++ b/salt/elasticfleet/files/integrations-dynamic/fleet-server/fleet-server.json @@ -0,0 +1,19 @@ +{ + "package": { + "name": "fleet_server", + "version": "" + }, + "name": "fleet_server-1", + "namespace": "default", + "policy_id": "FleetServer_hostname", + "vars": {}, + "inputs": { + "fleet_server-fleet-server": { + "enabled": true, + "vars": { + "custom": "server.ssl.supported_protocols: [\"TLSv1.2\", \"TLSv1.3\"]\nserver.ssl.cipher_suites: [ \"ECDHE-RSA-AES-128-GCM-SHA256\", \"ECDHE-RSA-AES-256-GCM-SHA384\", \"ECDHE-RSA-AES-128-CBC-SHA\", \"ECDHE-RSA-AES-256-CBC-SHA\", \"RSA-AES-128-GCM-SHA256\", \"RSA-AES-256-GCM-SHA384\"]" + }, + "streams": {} + } + } +} diff --git a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json index de35f803b8..15f08a1511 100644 --- a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json +++ b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json @@ -5,7 +5,7 @@ "package": { "name": "endpoint", "title": "Elastic Defend", - "version": "8.10.2" + "version": "8.14.0" }, "enabled": true, "policy_id": "endpoints-initial", diff --git a/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json b/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json index ac4394e628..ab7e0783fe 100644 --- a/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json +++ b/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json @@ -11,7 +11,7 @@ "winlogs-winlog": { "enabled": true, "streams": { - "winlog.winlog": { + "winlog.winlogs": { "enabled": true, "vars": { "channel": "Microsoft-Windows-Windows Defender/Operational", diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json index 3b1a50560e..fb8c310403 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json @@ -20,7 +20,7 @@ ], "data_stream.dataset": "import", "custom": "", - "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-1.43.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-1.38.0\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-1.43.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-1.43.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-1.38.0\n- add_fields:\n target: data_stream\n fields:\n dataset: import", + "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-1.59.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-1.45.1\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-1.59.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-1.59.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-1.45.1\n- add_fields:\n target: data_stream\n fields:\n dataset: import", "tags": [ "import" ] diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server new file mode 100644 index 0000000000..8f7c8b8b4f --- /dev/null +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-elastic-fleet-common + +# Get all the fleet policies +json_output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -L -X GET "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true') + +# Extract the IDs that start with "FleetServer_" +POLICY=$(echo "$json_output" | jq -r '.items[] | select(.id | startswith("FleetServer_")) | .id') + +# Iterate over each ID in the POLICY variable +for POLICYNAME in $POLICY; do + printf "\nUpdating Policy: $POLICYNAME\n" + + # First get the Integration ID + INTEGRATION_ID=$(/usr/sbin/so-elastic-fleet-agent-policy-view "$POLICYNAME" | jq -r '.item.package_policies[] | select(.package.name == "fleet_server") | .id') + + # Modify the default integration policy to update the policy_id and an with the correct naming + UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "$POLICYNAME" --arg name "fleet_server-$POLICYNAME" ' + .policy_id = $policy_id | + .name = $name' /opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json) + + # Now update the integration policy using the modified JSON + elastic_fleet_integration_update "$INTEGRATION_ID" "$UPDATED_INTEGRATION_POLICY" +done \ No newline at end of file diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load index 518d29d26f..26414a94bc 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load @@ -12,7 +12,10 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then # First, check for any package upgrades /usr/sbin/so-elastic-fleet-package-upgrade - # Second, configure Elastic Defend Integration seperately + # Second, update Fleet Server policies + /sbin/so-elastic-fleet-integration-policy-elastic-fleet-server + + # Third, configure Elastic Defend Integration seperately /usr/sbin/so-elastic-fleet-integration-policy-elastic-defend # Initial Endpoints diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index 0748557fd1..deb16dadfa 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -53,7 +53,8 @@ fi printf "\n### Create ES Token ###\n" ESTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/service_tokens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq -r .value) -### Create Outputs & Fleet URLs ### +### Create Outputs, Fleet Policy and Fleet URLs ### +# Create the Manager Elasticsearch Output first and set it as the default output printf "\nAdd Manager Elasticsearch Output...\n" ESCACRT=$(openssl x509 -in $INTCA) JSON_STRING=$( jq -n \ @@ -62,7 +63,21 @@ JSON_STRING=$( jq -n \ curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" printf "\n\n" -printf "\nCreate Logstash Output Config if node is not an Import or Eval install\n" +# Create the Manager Fleet Server Host Agent Policy +# This has to be done while the Elasticsearch Output is set to the default Output +printf "Create Manager Fleet Server Policy...\n" +elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "false" "120" + +# Modify the default integration policy to update the policy_id with the correct naming +UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "FleetServer_{{ GLOBALS.hostname }}" --arg name "fleet_server-{{ GLOBALS.hostname }}" ' +.policy_id = $policy_id | +.name = $name' /opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json) + +# Add the Fleet Server Integration to the new Fleet Policy +elastic_fleet_integration_create "$UPDATED_INTEGRATION_POLICY" + +# Now we can create the Logstash Output and set it to to be the default Output +printf "\n\nCreate Logstash Output Config if node is not an Import or Eval install\n" {% if grains.role not in ['so-import', 'so-eval'] %} LOGSTASHCRT=$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt) LOGSTASHKEY=$(openssl rsa -in /etc/pki/elasticfleet-logstash.key) @@ -101,16 +116,6 @@ printf "\n\n" # Load Elasticsearch templates /usr/sbin/so-elasticsearch-templates-load -# Manager Fleet Server Host -elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "true" "120" - -#Temp Fixup for ES Output bug -JSON_STRING=$( jq -n \ - --arg NAME "FleetServer_{{ GLOBALS.hostname }}" \ - '{"name": $NAME,"description": $NAME,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":120,"data_output_id":"so-manager_elasticsearch"}' - ) -curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/FleetServer_{{ GLOBALS.hostname }}" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" - # Initial Endpoints Policy elastic_fleet_policy_create "endpoints-initial" "Initial Endpoint Policy" "false" "1209600" @@ -165,4 +170,4 @@ salt-call state.apply elasticfleet queue=True # Generate installers & install Elastic Agent on the node so-elastic-agent-gen-installers salt-call state.apply elasticfleet.install_agent_grid queue=True -exit 0 \ No newline at end of file +exit 0 diff --git a/salt/elasticsearch/config.sls b/salt/elasticsearch/config.sls index 27a8a0fd66..a3dd189adf 100644 --- a/salt/elasticsearch/config.sls +++ b/salt/elasticsearch/config.sls @@ -118,6 +118,11 @@ esingestconf: - user: 930 - group: 939 +# Remove .fleet_final_pipeline-1 because we are using global@custom now +so-fleet-final-pipeline-remove: + file.absent: + - name: /opt/so/conf/elasticsearch/ingest/.fleet_final_pipeline-1 + # Auto-generate Elasticsearch ingest node pipelines from pillar {% for pipeline, config in ELASTICSEARCHMERGED.pipelines.items() %} es_ingest_conf_{{pipeline}}: diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 36f44ac07f..7201df25e7 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -56,87 +56,6 @@ elasticsearch: enabled: true key: /usr/share/elasticsearch/config/elasticsearch.key verification_mode: none - pipelines: - custom001: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom001 - - pipeline: - name: common - custom002: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom002 - - pipeline: - name: common - custom003: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom003 - - pipeline: - name: common - custom004: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom004 - - pipeline: - name: common - custom005: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom005 - - pipeline: - name: common - custom006: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom006 - - pipeline: - name: common - custom007: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom007 - - pipeline: - name: common - custom008: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom008 - - pipeline: - name: common - custom009: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom009 - - pipeline: - name: common - custom010: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom010 - - pipeline: - name: common index_settings: global_overrides: index_template: @@ -170,84 +89,13 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-items: - index_sorting: false - index_template: - composed_of: - - so-items-mappings - index_patterns: - - .items-default-** - priority: 500 - template: - mappings: - date_detection: false - settings: - index: - lifecycle: - name: so-items-logs - rollover_alias: ".items-default" - routing: - allocation: - include: - _tier_preference: "data_content" - mapping: - total_fields: - limit: 10000 - number_of_replicas: 0 - number_of_shards: 1 - refresh_interval: 30s - sort: - field: '@timestamp' - order: desc - policy: - phases: - hot: - actions: - rollover: - max_size: 50gb - min_age: 0ms - so-lists: - index_sorting: false - index_template: - composed_of: - - so-lists-mappings - index_patterns: - - .lists-default-** - priority: 500 - template: - mappings: - date_detection: false - settings: - index: - lifecycle: - name: so-lists-logs - rollover_alias: ".lists-default" - routing: - allocation: - include: - _tier_preference: "data_content" - mapping: - total_fields: - limit: 10000 - number_of_replicas: 0 - number_of_shards: 1 - refresh_interval: 30s - sort: - field: '@timestamp' - order: desc - policy: - phases: - hot: - actions: - rollover: - max_size: 50gb - min_age: 0ms so-case: index_sorting: false index_template: composed_of: - case-mappings - case-settings + ignore_missing_component_templates: [] index_patterns: - so-case* priority: 500 @@ -271,36 +119,7 @@ elasticsearch: sort: field: '@timestamp' order: desc - so-detection: - index_sorting: false - index_template: - composed_of: - - detection-mappings - - detection-settings - index_patterns: - - so-detection* - priority: 500 - template: - mappings: - date_detection: false - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - settings: - index: - mapping: - total_fields: - limit: 1500 - number_of_replicas: 0 - number_of_shards: 1 - refresh_interval: 1s - sort: - field: '@timestamp' - order: desc - so-logs-soc: + so-common: close: 30 delete: 365 index_sorting: false @@ -312,7 +131,9 @@ elasticsearch: - dtc-base-mappings - client-mappings - dtc-client-mappings + - cloud-mappings - container-mappings + - data_stream-mappings - destination-mappings - dtc-destination-mappings - pb-override-destination-mappings @@ -336,10 +157,12 @@ elasticsearch: - dtc-network-mappings - observer-mappings - dtc-observer-mappings + - orchestrator-mappings - organization-mappings - package-mappings - process-mappings - dtc-process-mappings + - registry-mappings - related-mappings - rule-mappings - dtc-rule-mappings @@ -349,17 +172,23 @@ elasticsearch: - source-mappings - dtc-source-mappings - pb-override-source-mappings + - syslog-mappings + - dtc-syslog-mappings - threat-mappings - tls-mappings + - tracing-mappings - url-mappings - user_agent-mappings - dtc-user_agent-mappings + - vulnerability-mappings - common-settings - common-dynamic-mappings + - winlog-mappings data_stream: {} + ignore_missing_component_templates: [] index_patterns: - - logs-soc-so* - priority: 500 + - logs-*-so* + priority: 1 template: mappings: date_detection: false @@ -372,7 +201,7 @@ elasticsearch: settings: index: lifecycle: - name: so-soc-logs + name: so-common-logs mapping: total_fields: limit: 5000 @@ -407,75 +236,16 @@ elasticsearch: priority: 50 min_age: 30d warm: 7 - so-common: - close: 30 - delete: 365 + so-detection: index_sorting: false index_template: composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - syslog-mappings - - dtc-syslog-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings - - winlog-mappings - data_stream: {} + - detection-mappings + - detection-settings + ignore_missing_component_templates: [] index_patterns: - - logs-*-so* - priority: 1 + - so-detection* + priority: 500 template: mappings: date_detection: false @@ -487,42 +257,15 @@ elasticsearch: match_mapping_type: string settings: index: - lifecycle: - name: so-common-logs mapping: total_fields: - limit: 5000 + limit: 1500 number_of_replicas: 0 number_of_shards: 1 refresh_interval: 30s sort: field: '@timestamp' order: desc - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - warm: 7 so-endgame: index_sorting: false index_template: @@ -585,6 +328,7 @@ elasticsearch: - common-settings - common-dynamic-mappings - winlog-mappings + ignore_missing_component_templates: [] index_patterns: - endgame* priority: 500 @@ -690,6 +434,7 @@ elasticsearch: - dtc-user_agent-mappings - common-settings - common-dynamic-mappings + ignore_missing_component_templates: [] index_patterns: - so-idh-* priority: 500 @@ -802,6 +547,7 @@ elasticsearch: - common-dynamic-mappings - winlog-mappings data_stream: {} + ignore_missing_component_templates: [] index_patterns: - logs-import-so* priority: 500 @@ -852,6 +598,91 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-items: + index_sorting: false + index_template: + composed_of: + - so-items-mappings + ignore_missing_component_templates: [] + index_patterns: + - .items-default-** + priority: 500 + template: + mappings: + date_detection: false + settings: + index: + lifecycle: + name: so-items-logs + rollover_alias: .items-default + mapping: + total_fields: + limit: 10000 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + routing: + allocation: + include: + _tier_preference: data_content + sort: + field: '@timestamp' + order: desc + policy: + phases: + hot: + actions: + rollover: + max_size: 50gb + min_age: 0ms + so-kismet: + index_sorting: false + index_template: + composed_of: + - kismet-mappings + - source-mappings + - client-mappings + - device-mappings + - network-mappings + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: [] + index_patterns: + - logs-kismet-so* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-kismet-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-kratos: close: 30 delete: 365 @@ -911,6 +742,7 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: [] index_patterns: - logs-kratos-so* priority: 500 @@ -950,17 +782,54 @@ elasticsearch: hot: actions: rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + warm: 7 + so-lists: + index_sorting: false + index_template: + composed_of: + - so-lists-mappings + ignore_missing_component_templates: [] + index_patterns: + - .lists-default-** + priority: 500 + template: + mappings: + date_detection: false + settings: + index: + lifecycle: + name: so-lists-logs + rollover_alias: .lists-default + mapping: + total_fields: + limit: 10000 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + routing: + allocation: + include: + _tier_preference: data_content + sort: + field: '@timestamp' + order: desc + policy: + phases: + hot: + actions: + rollover: + max_size: 50gb min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - warm: 7 so-logs: index_sorting: false index_template: @@ -973,6 +842,7 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: [] index_patterns: - logs-*-* priority: 225 @@ -1034,6 +904,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-1password.item_usages@custom index_patterns: - logs-1password.item_usages-* priority: 501 @@ -1078,6 +950,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-1password.signin_attempts@custom index_patterns: - logs-1password.signin_attempts-* priority: 501 @@ -1122,6 +996,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-apache.access@custom index_patterns: - logs-apache.access-* priority: 501 @@ -1166,6 +1042,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-apache.error@custom index_patterns: - logs-apache.error-* priority: 501 @@ -1210,6 +1088,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-auditd.log@custom index_patterns: - logs-auditd.log-* priority: 501 @@ -1254,6 +1134,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-auth0.logs@custom index_patterns: - logs-auth0.logs-* priority: 501 @@ -1288,25 +1170,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-aws_x_cloudfront_logs: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-aws.cloudfront_logs@package + - logs-aws.cloudfront_logs@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-aws.cloudfront_logs@custom index_patterns: - - "logs-aws.cloudfront_logs-*" + - logs-aws.cloudfront_logs-* + priority: 501 template: settings: index: lifecycle: name: so-logs-aws.cloudfront_logs-logs number_of_replicas: 0 - composed_of: - - "logs-aws.cloudfront_logs@package" - - "logs-aws.cloudfront_logs@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -1342,6 +1226,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.cloudtrail@custom index_patterns: - logs-aws.cloudtrail-* priority: 501 @@ -1386,6 +1272,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.cloudwatch_logs@custom index_patterns: - logs-aws.cloudwatch_logs-* priority: 501 @@ -1430,6 +1318,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.ec2_logs@custom index_patterns: - logs-aws.ec2_logs-* priority: 501 @@ -1474,6 +1364,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.elb_logs@custom index_patterns: - logs-aws.elb_logs-* priority: 501 @@ -1518,6 +1410,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.firewall_logs@custom index_patterns: - logs-aws.firewall_logs-* priority: 501 @@ -1552,25 +1446,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-aws_x_guardduty: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-aws.guardduty@package + - logs-aws.guardduty@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-aws.guardduty@custom index_patterns: - - "logs-aws.guardduty-*" + - logs-aws.guardduty-* + priority: 501 template: settings: index: lifecycle: name: so-logs-aws.guardduty-logs number_of_replicas: 0 - composed_of: - - "logs-aws.guardduty@package" - - "logs-aws.guardduty@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -1596,25 +1492,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-aws_x_inspector: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-aws.inspector@package + - logs-aws.inspector@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-aws.inspector@custom index_patterns: - - "logs-aws.inspector-*" + - logs-aws.inspector-* + priority: 501 template: settings: index: lifecycle: name: so-logs-aws.inspector-logs number_of_replicas: 0 - composed_of: - - "logs-aws.inspector@package" - - "logs-aws.inspector@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -1650,6 +1548,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.route53_public_logs@custom index_patterns: - logs-aws.route53_public_logs-* priority: 501 @@ -1694,6 +1594,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.route53_resolver_logs@custom index_patterns: - logs-aws.route53_resolver_logs-* priority: 501 @@ -1738,6 +1640,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.s3access@custom index_patterns: - logs-aws.s3access-* priority: 501 @@ -1772,25 +1676,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-aws_x_securityhub_findings: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-aws.securityhub_findings@package + - logs-aws.securityhub_findings@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-aws.securityhub_findings@custom index_patterns: - - "logs-aws.securityhub_findings-*" + - logs-aws.securityhub_findings-* + priority: 501 template: settings: index: lifecycle: name: so-logs-aws.securityhub_findings-logs number_of_replicas: 0 - composed_of: - - "logs-aws.securityhub_findings@package" - - "logs-aws.securityhub_findings@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -1816,25 +1722,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-aws_x_securityhub_insights: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-aws.securityhub_insights@package + - logs-aws.securityhub_insights@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-aws.securityhub_insights@custom index_patterns: - - "logs-aws.securityhub_insights-*" + - logs-aws.securityhub_insights-* + priority: 501 template: settings: index: lifecycle: name: so-logs-aws.securityhub_insights-logs number_of_replicas: 0 - composed_of: - - "logs-aws.securityhub_insights@package" - - "logs-aws.securityhub_insights@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -1870,6 +1778,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.vpcflow@custom index_patterns: - logs-aws.vpcflow-* priority: 501 @@ -1914,6 +1824,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.waf@custom index_patterns: - logs-aws.waf-* priority: 501 @@ -1958,6 +1870,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.activitylogs@custom index_patterns: - logs-azure.activitylogs-* priority: 501 @@ -2002,6 +1916,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.application_gateway@custom index_patterns: - logs-azure.application_gateway-* priority: 501 @@ -2046,6 +1962,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.auditlogs@custom index_patterns: - logs-azure.auditlogs-* priority: 501 @@ -2090,6 +2008,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.eventhub@custom index_patterns: - logs-azure.eventhub-* priority: 501 @@ -2134,6 +2054,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.firewall_logs@custom index_patterns: - logs-azure.firewall_logs-* priority: 501 @@ -2178,6 +2100,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.identity_protection@custom index_patterns: - logs-azure.identity_protection-* priority: 501 @@ -2222,6 +2146,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.platformlogs@custom index_patterns: - logs-azure.platformlogs-* priority: 501 @@ -2266,6 +2192,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.provisioning@custom index_patterns: - logs-azure.provisioning-* priority: 501 @@ -2310,6 +2238,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.signinlogs@custom index_patterns: - logs-azure.signinlogs-* priority: 501 @@ -2354,6 +2284,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.springcloudlogs@custom index_patterns: - logs-azure.springcloudlogs-* priority: 501 @@ -2398,6 +2330,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-barracuda.waf@custom index_patterns: - logs-barracuda.waf-* priority: 501 @@ -2442,6 +2376,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-carbonblack_edr.log@custom index_patterns: - logs-carbonblack_edr.log-* priority: 501 @@ -2476,25 +2412,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-cef_x_log: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-cef.log@package + - logs-cef.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-cef.log@custom index_patterns: - - "logs-cef.log-*" + - logs-cef.log-* + priority: 501 template: settings: index: lifecycle: name: so-logs-cef.log-logs number_of_replicas: 0 - composed_of: - - "logs-cef.log@package" - - "logs-cef.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -2520,25 +2458,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-checkpoint_x_firewall: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-checkpoint.firewall@package + - logs-checkpoint.firewall@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-checkpoint.firewall@custom index_patterns: - - "logs-checkpoint.firewall-*" + - logs-checkpoint.firewall-* + priority: 501 template: settings: index: lifecycle: name: so-logs-checkpoint.firewall-logs number_of_replicas: 0 - composed_of: - - "logs-checkpoint.firewall@package" - - "logs-checkpoint.firewall@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -2574,6 +2514,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_asa.log@custom index_patterns: - logs-cisco_asa.log-* priority: 501 @@ -2618,6 +2560,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_duo.admin@custom index_patterns: - logs-cisco_duo.admin-* priority: 501 @@ -2662,6 +2606,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_duo.auth@custom index_patterns: - logs-cisco_duo.auth-* priority: 501 @@ -2706,6 +2652,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_duo.offline_enrollment@custom index_patterns: - logs-cisco_duo.offline_enrollment-* priority: 501 @@ -2750,6 +2698,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_duo.summary@custom index_patterns: - logs-cisco_duo.summary-* priority: 501 @@ -2794,6 +2744,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_duo.telephony@custom index_patterns: - logs-cisco_duo.telephony-* priority: 501 @@ -2828,25 +2780,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-cisco_ftd_x_log: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-cisco_ftd.log@package + - logs-cisco_ftd.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-cisco_ftd.log@custom index_patterns: - - "logs-cisco_ftd.log-*" + - logs-cisco_ftd.log-* + priority: 501 template: settings: index: lifecycle: name: so-logs-cisco_ftd.log-logs number_of_replicas: 0 - composed_of: - - "logs-cisco_ftd.log@package" - - "logs-cisco_ftd.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -2872,25 +2826,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-cisco_ios_x_log: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-cisco_ios.log@package + - logs-cisco_ios.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-cisco_ios.log@custom index_patterns: - - "logs-cisco_ios.log-*" + - logs-cisco_ios.log-* + priority: 501 template: settings: index: lifecycle: name: so-logs-cisco_ios.log-logs number_of_replicas: 0 - composed_of: - - "logs-cisco_ios.log@package" - - "logs-cisco_ios.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -2916,25 +2872,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-cisco_ise_x_log: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-cisco_ise.log@package + - logs-cisco_ise.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-cisco_ise.log@custom index_patterns: - - "logs-cisco_ise.log-*" + - logs-cisco_ise.log-* + priority: 501 template: settings: index: lifecycle: name: so-logs-cisco_ise.log-logs number_of_replicas: 0 - composed_of: - - "logs-cisco_ise.log@package" - - "logs-cisco_ise.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -2970,6 +2928,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_meraki.events@custom index_patterns: - logs-cisco_meraki.events-* priority: 501 @@ -3014,6 +2974,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_meraki.log@custom index_patterns: - logs-cisco_meraki.log-* priority: 501 @@ -3058,6 +3020,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_umbrella.log@custom index_patterns: - logs-cisco_umbrella.log-* priority: 501 @@ -3092,25 +3056,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_adc_x_interface: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-citrix_adc.interface@package + - logs-citrix_adc.interface@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-citrix_adc.interface@custom index_patterns: - - "logs-citrix_adc.interface-*" + - logs-citrix_adc.interface-* + priority: 501 template: settings: index: lifecycle: name: so-logs-citrix_adc.interface-logs number_of_replicas: 0 - composed_of: - - "logs-citrix_adc.interface@package" - - "logs-citrix_adc.interface@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -3136,25 +3102,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_adc_x_lbvserver: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-citrix_adc.lbvserver@package + - logs-citrix_adc.lbvserver@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-citrix_adc.lbvserver@custom index_patterns: - - "logs-citrix_adc.lbvserver-*" + - logs-citrix_adc.lbvserver-* + priority: 501 template: settings: index: lifecycle: name: so-logs-citrix_adc.lbvserver-logs number_of_replicas: 0 - composed_of: - - "logs-citrix_adc.lbvserver@package" - - "logs-citrix_adc.lbvserver@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -3180,25 +3148,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_adc_x_service: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-citrix_adc.service@package + - logs-citrix_adc.service@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-citrix_adc.service@custom index_patterns: - - "logs-citrix_adc.service-*" + - logs-citrix_adc.service-* + priority: 501 template: settings: index: lifecycle: name: so-logs-citrix_adc.service-logs number_of_replicas: 0 - composed_of: - - "logs-citrix_adc.service@package" - - "logs-citrix_adc.service@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -3224,25 +3194,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_adc_x_system: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-citrix_adc.system@package + - logs-citrix_adc.system@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-citrix_adc.system@custom index_patterns: - - "logs-citrix_adc.system-*" + - logs-citrix_adc.system-* + priority: 501 template: settings: index: lifecycle: name: so-logs-citrix_adc.system-logs number_of_replicas: 0 - composed_of: - - "logs-citrix_adc.system@package" - - "logs-citrix_adc.system@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -3268,25 +3240,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_adc_x_vpn: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-citrix_adc.vpn@package + - logs-citrix_adc.vpn@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-citrix_adc.vpn@custom index_patterns: - - "logs-citrix_adc.vpn-*" + - logs-citrix_adc.vpn-* + priority: 501 template: settings: index: lifecycle: name: so-logs-citrix_adc.vpn-logs number_of_replicas: 0 - composed_of: - - "logs-citrix_adc.vpn@package" - - "logs-citrix_adc.vpn@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -3312,25 +3286,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_waf_x_log: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-citrix_waf.log@package + - logs-citrix_waf.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-citrix_waf.log@custom index_patterns: - - "logs-citrix_waf.log-*" + - logs-citrix_waf.log-* + priority: 501 template: settings: index: lifecycle: name: so-logs-citrix_waf.log-logs number_of_replicas: 0 - composed_of: - - "logs-citrix_waf.log@package" - - "logs-citrix_waf.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -3366,6 +3342,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cloudflare.audit@custom index_patterns: - logs-cloudflare.audit-* priority: 501 @@ -3410,6 +3388,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cloudflare.logpull@custom index_patterns: - logs-cloudflare.logpull-* priority: 501 @@ -3454,6 +3434,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-crowdstrike.falcon@custom index_patterns: - logs-crowdstrike.falcon-* priority: 501 @@ -3498,6 +3480,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-crowdstrike.fdr@custom index_patterns: - logs-crowdstrike.fdr-* priority: 501 @@ -3542,6 +3526,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-darktrace.ai_analyst_alert@custom index_patterns: - logs-darktrace.ai_analyst_alert-* priority: 501 @@ -3586,6 +3572,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-darktrace.model_breach_alert@custom index_patterns: - logs-darktrace.model_breach_alert-* priority: 501 @@ -3630,6 +3618,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-darktrace.system_status_alert@custom index_patterns: - logs-darktrace.system_status_alert-* priority: 501 @@ -3675,6 +3665,7 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: [] index_patterns: - logs-detections.alerts-* priority: 501 @@ -3737,6 +3728,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent@custom index_patterns: - logs-elastic_agent-* priority: 501 @@ -3798,6 +3791,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent.apm_server@custom index_patterns: - logs-elastic_agent.apm_server-* priority: 501 @@ -3859,6 +3854,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent.auditbeat@custom index_patterns: - logs-elastic_agent.auditbeat-* priority: 501 @@ -3917,6 +3914,8 @@ elasticsearch: - logs-elastic_agent.cloudbeat@custom - so-fleet_globals-1 - so-fleet_agent_id_verification-1 + ignore_missing_component_templates: + - logs-elastic_agent.cloudbeat@custom index_patterns: - logs-elastic_agent.cloudbeat-* priority: 501 @@ -3979,6 +3978,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent.endpoint_security@custom index_patterns: - logs-elastic_agent.endpoint_security-* priority: 501 @@ -4035,6 +4036,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent.filebeat@custom index_patterns: - logs-elastic_agent.filebeat-* priority: 501 @@ -4091,6 +4094,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent.fleet_server@custom index_patterns: - logs-elastic_agent.fleet_server-* priority: 501 @@ -4140,6 +4145,8 @@ elasticsearch: - logs-elastic_agent.heartbeat@custom - so-fleet_globals-1 - so-fleet_agent_id_verification-1 + ignore_missing_component_templates: + - logs-elastic_agent.heartbeat@custom index_patterns: - logs-elastic_agent.heartbeat-* priority: 501 @@ -4202,6 +4209,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent.metricbeat@custom index_patterns: - logs-elastic_agent.metricbeat-* priority: 501 @@ -4258,6 +4267,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent.osquerybeat@custom index_patterns: - logs-elastic_agent.osquerybeat-* priority: 501 @@ -4313,6 +4324,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent.packetbeat@custom index_patterns: - logs-elastic_agent.packetbeat-* priority: 501 @@ -4375,6 +4388,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.alerts@custom index_patterns: - logs-endpoint.alerts-* priority: 501 @@ -4431,6 +4446,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.diagnostic.collection@custom index_patterns: - .logs-endpoint.diagnostic.collection-* priority: 501 @@ -4487,6 +4504,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.events.api@custom index_patterns: - logs-endpoint.events.api-* priority: 501 @@ -4543,6 +4562,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.events.file@custom index_patterns: - logs-endpoint.events.file-* priority: 501 @@ -4599,6 +4620,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.events.library@custom index_patterns: - logs-endpoint.events.library-* priority: 501 @@ -4655,6 +4678,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.events.network@custom index_patterns: - logs-endpoint.events.network-* priority: 501 @@ -4711,6 +4736,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.events.process@custom index_patterns: - logs-endpoint.events.process-* priority: 501 @@ -4767,6 +4794,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.events.registry@custom index_patterns: - logs-endpoint.events.registry-* priority: 501 @@ -4823,6 +4852,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.events.security@custom index_patterns: - logs-endpoint.events.security-* priority: 501 @@ -4878,6 +4909,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-f5_bigip.log@custom index_patterns: - logs-f5_bigip.log-* priority: 501 @@ -4922,6 +4955,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-fim.event@custom index_patterns: - logs-fim.event-* priority: 501 @@ -4966,6 +5001,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-fireeye.nx@custom index_patterns: - logs-fireeye.nx-* priority: 501 @@ -5010,6 +5047,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-fortinet_fortigate.log@custom index_patterns: - logs-fortinet_fortigate.log-* priority: 501 @@ -5054,6 +5093,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-fortinet.clientendpoint@custom index_patterns: - logs-fortinet.clientendpoint-* priority: 501 @@ -5098,6 +5139,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-fortinet.firewall@custom index_patterns: - logs-fortinet.firewall-* priority: 501 @@ -5142,6 +5185,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-fortinet.fortimail@custom index_patterns: - logs-fortinet.fortimail-* priority: 501 @@ -5186,6 +5231,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-fortinet.fortimanager@custom index_patterns: - logs-fortinet.fortimanager-* priority: 501 @@ -5230,6 +5277,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-gcp.audit@custom index_patterns: - logs-gcp.audit-* priority: 501 @@ -5274,6 +5323,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-gcp.dns@custom index_patterns: - logs-gcp.dns-* priority: 501 @@ -5318,6 +5369,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-gcp.firewall@custom index_patterns: - logs-gcp.firewall-* priority: 501 @@ -5362,6 +5415,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-gcp.loadbalancing_logs@custom index_patterns: - logs-gcp.loadbalancing_logs-* priority: 501 @@ -5406,6 +5461,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-gcp.vpcflow@custom index_patterns: - logs-gcp.vpcflow-* priority: 501 @@ -5450,6 +5507,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-github.audit@custom index_patterns: - logs-github.audit-* priority: 501 @@ -5494,6 +5553,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-github.code_scanning@custom index_patterns: - logs-github.code_scanning-* priority: 501 @@ -5538,6 +5599,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-github.dependabot@custom index_patterns: - logs-github.dependabot-* priority: 501 @@ -5582,6 +5645,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-github.issues@custom index_patterns: - logs-github.issues-* priority: 501 @@ -5626,6 +5691,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-github.secret_scanning@custom index_patterns: - logs-github.secret_scanning-* priority: 501 @@ -5670,6 +5737,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.access_transparency@custom index_patterns: - logs-google_workspace.access_transparency-* priority: 501 @@ -5714,6 +5783,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.admin@custom index_patterns: - logs-google_workspace.admin-* priority: 501 @@ -5758,6 +5829,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.alert@custom index_patterns: - logs-google_workspace.alert-* priority: 501 @@ -5802,6 +5875,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.context_aware_access@custom index_patterns: - logs-google_workspace.context_aware_access-* priority: 501 @@ -5846,6 +5921,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.device@custom index_patterns: - logs-google_workspace.device-* priority: 501 @@ -5890,6 +5967,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.drive@custom index_patterns: - logs-google_workspace.drive-* priority: 501 @@ -5934,6 +6013,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.gcp@custom index_patterns: - logs-google_workspace.gcp-* priority: 501 @@ -5978,6 +6059,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.group_enterprise@custom index_patterns: - logs-google_workspace.group_enterprise-* priority: 501 @@ -6022,6 +6105,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.groups@custom index_patterns: - logs-google_workspace.groups-* priority: 501 @@ -6066,6 +6151,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.login@custom index_patterns: - logs-google_workspace.login-* priority: 501 @@ -6110,6 +6197,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.rules@custom index_patterns: - logs-google_workspace.rules-* priority: 501 @@ -6154,6 +6243,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.saml@custom index_patterns: - logs-google_workspace.saml-* priority: 501 @@ -6198,6 +6289,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.token@custom index_patterns: - logs-google_workspace.token-* priority: 501 @@ -6242,6 +6335,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.user_accounts@custom index_patterns: - logs-google_workspace.user_accounts-* priority: 501 @@ -6286,6 +6381,9 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-http_endpoint.generic@package + - logs-http_endpoint.generic@custom index_patterns: - logs-http_endpoint.generic-* priority: 501 @@ -6330,6 +6428,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-httpjson.generic@custom index_patterns: - logs-httpjson.generic-* priority: 501 @@ -6364,25 +6464,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-iis_x_access: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-iis.access@package + - logs-iis.access@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-iis.access@custom index_patterns: - - "logs-iis.access-*" + - logs-iis.access-* + priority: 501 template: settings: index: lifecycle: name: so-logs-iis.access-logs number_of_replicas: 0 - composed_of: - - "logs-iis.access@package" - - "logs-iis.access@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -6408,25 +6510,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-iis_x_error: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-iis.error@package + - logs-iis.error@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-iis.error@custom index_patterns: - - "logs-iis.error-*" + - logs-iis.error-* + priority: 501 template: settings: index: lifecycle: name: so-logs-iis.error-logs number_of_replicas: 0 - composed_of: - - "logs-iis.error@package" - - "logs-iis.error@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -6462,6 +6566,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-juniper_srx.log@custom index_patterns: - logs-juniper_srx.log-* priority: 501 @@ -6506,6 +6612,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-juniper.junos@custom index_patterns: - logs-juniper.junos-* priority: 501 @@ -6550,6 +6658,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-juniper.netscreen@custom index_patterns: - logs-juniper.netscreen-* priority: 501 @@ -6594,6 +6704,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-juniper.srx@custom index_patterns: - logs-juniper.srx-* priority: 501 @@ -6638,6 +6750,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-kafka_log.generic@custom index_patterns: - logs-kafka_log.generic-* priority: 501 @@ -6682,6 +6796,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-lastpass.detailed_shared_folder@custom index_patterns: - logs-lastpass.detailed_shared_folder-* priority: 501 @@ -6726,6 +6842,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-lastpass.event_report@custom index_patterns: - logs-lastpass.event_report-* priority: 501 @@ -6770,6 +6888,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-lastpass.user@custom index_patterns: - logs-lastpass.user-* priority: 501 @@ -6814,6 +6934,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-m365_defender.event@custom index_patterns: - logs-m365_defender.event-* priority: 501 @@ -6858,6 +6980,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-m365_defender.incident@custom index_patterns: - logs-m365_defender.incident-* priority: 501 @@ -6902,6 +7026,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-m365_defender.log@custom index_patterns: - logs-m365_defender.log-* priority: 501 @@ -6946,6 +7072,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-microsoft_defender_endpoint.log@custom index_patterns: - logs-microsoft_defender_endpoint.log-* priority: 501 @@ -6990,6 +7118,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-microsoft_dhcp.log@custom index_patterns: - logs-microsoft_dhcp.log-* priority: 501 @@ -7024,113 +7154,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-microsoft_sqlserver_x_audit: - index_sorting: False + index_sorting: false index_template: - index_patterns: - - "logs-microsoft_sqlserver.audit-*" - template: - settings: - index: - lifecycle: - name: so-logs-microsoft_sqlserver.audit-logs - number_of_replicas: 0 composed_of: - - "logs-microsoft_sqlserver.audit@package" - - "logs-microsoft_sqlserver.audit@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 + - logs-microsoft_sqlserver.audit@package + - logs-microsoft_sqlserver.audit@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 data_stream: - hidden: false allow_custom_routing: false - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-logs-microsoft_sqlserver_x_log: - index_sorting: False - index_template: - index_patterns: - - "logs-microsoft_sqlserver.log-*" - template: - settings: - index: - lifecycle: - name: so-logs-microsoft_sqlserver.log-logs - number_of_replicas: 0 - composed_of: - - "logs-microsoft_sqlserver.log@package" - - "logs-microsoft_sqlserver.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: hidden: false - allow_custom_routing: false - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-logs-mysql_x_error: - index_sorting: False - index_template: + ignore_missing_component_templates: + - logs-microsoft_sqlserver.audit@custom index_patterns: - - "logs-mysql.error-*" + - logs-microsoft_sqlserver.audit-* + priority: 501 template: settings: index: lifecycle: - name: so-logs-mysql.error-logs + name: so-logs-microsoft_sqlserver.audit-logs number_of_replicas: 0 - composed_of: - - "logs-mysql.error@package" - - "logs-mysql.error@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -7155,26 +7199,28 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-logs-mysql_x_slowlog: - index_sorting: False + so-logs-microsoft_sqlserver_x_log: + index_sorting: false index_template: + composed_of: + - logs-microsoft_sqlserver.log@package + - logs-microsoft_sqlserver.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-microsoft_sqlserver.log@custom index_patterns: - - "logs-mysql.slowlog-*" + - logs-microsoft_sqlserver.log-* + priority: 501 template: settings: index: lifecycle: - name: so-logs-mysql.slowlog-logs + name: so-logs-microsoft_sqlserver.log-logs number_of_replicas: 0 - composed_of: - - "logs-mysql.slowlog@package" - - "logs-mysql.slowlog@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -7210,6 +7256,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-mimecast.audit_events@custom index_patterns: - logs-mimecast.audit_events-* priority: 501 @@ -7254,6 +7302,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-mimecast.dlp_logs@custom index_patterns: - logs-mimecast.dlp_logs-* priority: 501 @@ -7298,6 +7348,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-mimecast.siem_logs@custom index_patterns: - logs-mimecast.siem_logs-* priority: 501 @@ -7342,6 +7394,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-mimecast.threat_intel_malware_customer@custom index_patterns: - logs-mimecast.threat_intel_malware_customer-* priority: 501 @@ -7386,6 +7440,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-mimecast.threat_intel_malware_grid@custom index_patterns: - logs-mimecast.threat_intel_malware_grid-* priority: 501 @@ -7430,6 +7486,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-mimecast.ttp_ap_logs@custom index_patterns: - logs-mimecast.ttp_ap_logs-* priority: 501 @@ -7474,6 +7532,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-mimecast.ttp_ip_logs@custom index_patterns: - logs-mimecast.ttp_ip_logs-* priority: 501 @@ -7518,6 +7578,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-mimecast.ttp_url_logs@custom index_patterns: - logs-mimecast.ttp_url_logs-* priority: 501 @@ -7551,25 +7613,27 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-logs-netflow_x_log: + so-logs-mysql_x_error: index_sorting: false index_template: composed_of: - - logs-netflow.log@package - - logs-netflow.log@custom + - logs-mysql.error@package + - logs-mysql.error@custom - so-fleet_globals-1 - so-fleet_agent_id_verification-1 data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-mysql.error@custom index_patterns: - - logs-netflow.log-* + - logs-mysql.error-* priority: 501 template: settings: index: lifecycle: - name: so-logs-netflow.log-logs + name: so-logs-mysql.error-logs number_of_replicas: 0 policy: phases: @@ -7595,26 +7659,28 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-logs-nginx_x_access: - index_sorting: False + so-logs-mysql_x_slowlog: + index_sorting: false index_template: + composed_of: + - logs-mysql.slowlog@package + - logs-mysql.slowlog@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-mysql.slowlog@custom index_patterns: - - "logs-nginx.access-*" + - logs-mysql.slowlog-* + priority: 501 template: settings: index: lifecycle: - name: so-logs-nginx.access-logs + name: so-logs-mysql.slowlog-logs number_of_replicas: 0 - composed_of: - - "logs-nginx.access@package" - - "logs-nginx.access@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -7639,26 +7705,28 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-logs-nginx_x_error: - index_sorting: False + so-logs-netflow_x_log: + index_sorting: false index_template: + composed_of: + - logs-netflow.log@package + - logs-netflow.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-netflow.log@custom index_patterns: - - "logs-nginx.error-*" + - logs-netflow.log-* + priority: 501 template: settings: index: lifecycle: - name: so-logs-nginx.error-logs + name: so-logs-netflow.log-logs number_of_replicas: 0 - composed_of: - - "logs-nginx.error@package" - - "logs-nginx.error@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -7683,26 +7751,74 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-metrics-nginx_x_stubstatus: - index_sorting: False + so-logs-nginx_x_access: + index_sorting: false index_template: + composed_of: + - logs-nginx.access@package + - logs-nginx.access@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-nginx.access@custom index_patterns: - - "metrics-nginx.stubstatus-*" + - logs-nginx.access-* + priority: 501 template: settings: index: lifecycle: - name: so-metrics-nginx.stubstatus-logs + name: so-logs-nginx.access-logs number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-nginx_x_error: + index_sorting: false + index_template: composed_of: - - "metrics-nginx.stubstatus@package" - - "metrics-nginx.stubstatus@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 + - logs-nginx.error@package + - logs-nginx.error@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 data_stream: - hidden: false allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-nginx.error@custom + index_patterns: + - logs-nginx.error-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-nginx.error-logs + number_of_replicas: 0 policy: phases: cold: @@ -7738,6 +7854,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-o365.audit@custom index_patterns: - logs-o365.audit-* priority: 501 @@ -7782,6 +7900,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-okta.system@custom index_patterns: - logs-okta.system-* priority: 501 @@ -7825,6 +7945,7 @@ elasticsearch: name: elastic_agent composed_of: - logs-osquery_manager.action.responses + ignore_missing_component_templates: [] index_patterns: - .logs-osquery_manager.action.responses* priority: 501 @@ -7842,6 +7963,7 @@ elasticsearch: name: elastic_agent composed_of: - logs-osquery_manager.actions + ignore_missing_component_templates: [] index_patterns: - .logs-osquery_manager.actions* priority: 501 @@ -7860,6 +7982,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-panw.panos@custom index_patterns: - logs-panw.panos-* priority: 501 @@ -7904,6 +8028,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-pfsense.log@custom index_patterns: - logs-pfsense.log-* priority: 501 @@ -7938,25 +8064,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-proofpoint_tap_x_clicks_blocked: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-proofpoint_tap.clicks_blocked@package + - logs-proofpoint_tap.clicks_blocked@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-proofpoint_tap.clicks_blocked@custom index_patterns: - - "logs-proofpoint_tap.clicks_blocked-*" + - logs-proofpoint_tap.clicks_blocked-* + priority: 501 template: settings: index: lifecycle: name: so-logs-proofpoint_tap.clicks_blocked-logs number_of_replicas: 0 - composed_of: - - "logs-proofpoint_tap.clicks_blocked@package" - - "logs-proofpoint_tap.clicks_blocked@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -7982,25 +8110,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-proofpoint_tap_x_clicks_permitted: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-proofpoint_tap.clicks_permitted@package + - logs-proofpoint_tap.clicks_permitted@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-proofpoint_tap.clicks_permitted@custom index_patterns: - - "logs-proofpoint_tap.clicks_permitted-*" + - logs-proofpoint_tap.clicks_permitted-* + priority: 501 template: settings: index: lifecycle: name: so-logs-proofpoint_tap.clicks_permitted-logs number_of_replicas: 0 - composed_of: - - "logs-proofpoint_tap.clicks_permitted@package" - - "logs-proofpoint_tap.clicks_permitted@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -8026,25 +8156,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-proofpoint_tap_x_message_blocked: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-proofpoint_tap.message_blocked@package + - logs-proofpoint_tap.message_blocked@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-proofpoint_tap.message_blocked@custom index_patterns: - - "logs-proofpoint_tap.message_blocked-*" + - logs-proofpoint_tap.message_blocked-* + priority: 501 template: settings: index: lifecycle: name: so-logs-proofpoint_tap.message_blocked-logs number_of_replicas: 0 - composed_of: - - "logs-proofpoint_tap.message_blocked@package" - - "logs-proofpoint_tap.message_blocked@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -8070,25 +8202,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-proofpoint_tap_x_message_delivered: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-proofpoint_tap.message_delivered@package + - logs-proofpoint_tap.message_delivered@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-proofpoint_tap.message_delivered@custom index_patterns: - - "logs-proofpoint_tap.message_delivered-*" + - logs-proofpoint_tap.message_delivered-* + priority: 501 template: settings: index: lifecycle: name: so-logs-proofpoint_tap.message_delivered-logs number_of_replicas: 0 - composed_of: - - "logs-proofpoint_tap.message_delivered@package" - - "logs-proofpoint_tap.message_delivered@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -8124,6 +8258,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-pulse_connect_secure.log@custom index_patterns: - logs-pulse_connect_secure.log-* priority: 501 @@ -8168,6 +8304,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sentinel_one.activity@custom index_patterns: - logs-sentinel_one.activity-* priority: 501 @@ -8212,6 +8350,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sentinel_one.agent@custom index_patterns: - logs-sentinel_one.agent-* priority: 501 @@ -8256,6 +8396,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sentinel_one.alert@custom index_patterns: - logs-sentinel_one.alert-* priority: 501 @@ -8300,6 +8442,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sentinel_one.group@custom index_patterns: - logs-sentinel_one.group-* priority: 501 @@ -8344,6 +8488,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sentinel_one.threat@custom index_patterns: - logs-sentinel_one.threat-* priority: 501 @@ -8378,25 +8524,73 @@ elasticsearch: priority: 50 min_age: 30d so-logs-snort_x_log: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-snort.log@package + - logs-snort.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-snort.log@custom index_patterns: - - "logs-snort.log-*" + - logs-snort.log-* + priority: 501 template: settings: index: lifecycle: name: so-logs-snort.log-logs number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-snyk_x_audit: + index_sorting: false + index_template: composed_of: - - "logs-snort.log@package" - - "logs-snort.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 + - logs-snyk.audit@package + - logs-snyk.audit@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 data_stream: - hidden: false allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-snyk.audit@custom + index_patterns: + - logs-snyk.audit-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-snyk.audit-logs + number_of_replicas: 0 policy: phases: cold: @@ -8421,25 +8615,27 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-logs-snyk_x_audit: + so-logs-snyk_x_vulnerabilities: index_sorting: false index_template: composed_of: - - logs-snyk.audit@package - - logs-snyk.audit@custom + - logs-snyk.vulnerabilities@package + - logs-snyk.vulnerabilities@custom - so-fleet_globals-1 - so-fleet_agent_id_verification-1 data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-snyk.vulnerabilities@custom index_patterns: - - logs-snyk.audit-* + - logs-snyk.vulnerabilities-* priority: 501 template: settings: index: lifecycle: - name: so-logs-snyk.audit-logs + name: so-logs-snyk.vulnerabilities-logs number_of_replicas: 0 policy: phases: @@ -8465,26 +8661,89 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-logs-snyk_x_vulnerabilities: + so-logs-soc: + close: 30 + delete: 365 index_sorting: false index_template: composed_of: - - logs-snyk.vulnerabilities@package - - logs-snyk.vulnerabilities@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false + - agent-mappings + - dtc-agent-mappings + - base-mappings + - dtc-base-mappings + - client-mappings + - dtc-client-mappings + - container-mappings + - destination-mappings + - dtc-destination-mappings + - pb-override-destination-mappings + - dll-mappings + - dns-mappings + - dtc-dns-mappings + - ecs-mappings + - dtc-ecs-mappings + - error-mappings + - event-mappings + - dtc-event-mappings + - file-mappings + - dtc-file-mappings + - group-mappings + - host-mappings + - dtc-host-mappings + - http-mappings + - dtc-http-mappings + - log-mappings + - network-mappings + - dtc-network-mappings + - observer-mappings + - dtc-observer-mappings + - organization-mappings + - package-mappings + - process-mappings + - dtc-process-mappings + - related-mappings + - rule-mappings + - dtc-rule-mappings + - server-mappings + - service-mappings + - dtc-service-mappings + - source-mappings + - dtc-source-mappings + - pb-override-source-mappings + - threat-mappings + - tls-mappings + - url-mappings + - user_agent-mappings + - dtc-user_agent-mappings + - common-settings + - common-dynamic-mappings + data_stream: {} + ignore_missing_component_templates: [] index_patterns: - - logs-snyk.vulnerabilities-* - priority: 501 + - logs-soc-so* + priority: 500 template: + mappings: + date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string settings: index: lifecycle: - name: so-logs-snyk.vulnerabilities-logs + name: so-logs-soc-logs + mapping: + total_fields: + limit: 5000 number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc policy: phases: cold: @@ -8509,6 +8768,7 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + warm: 7 so-logs-sonicwall_firewall_x_log: index_sorting: false index_template: @@ -8520,6 +8780,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sonicwall_firewall.log@custom index_patterns: - logs-sonicwall_firewall.log-* priority: 501 @@ -8564,6 +8826,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sophos_central.alert@custom index_patterns: - logs-sophos_central.alert-* priority: 501 @@ -8608,6 +8872,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sophos_central.event@custom index_patterns: - logs-sophos_central.event-* priority: 501 @@ -8652,6 +8918,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sophos.utm@custom index_patterns: - logs-sophos.utm-* priority: 501 @@ -8696,6 +8964,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sophos.xg@custom index_patterns: - logs-sophos.xg-* priority: 501 @@ -8740,6 +9010,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-symantec_endpoint.log@custom index_patterns: - logs-symantec_endpoint.log-* priority: 501 @@ -8785,6 +9057,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-system.application@custom index_patterns: - logs-system.application* priority: 501 @@ -8827,9 +9101,12 @@ elasticsearch: - logs-system.auth@custom - so-fleet_globals-1 - so-fleet_agent_id_verification-1 + - so-system-mappings data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-system.auth@custom index_patterns: - logs-system.auth* priority: 501 @@ -8875,6 +9152,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-system.security@custom index_patterns: - logs-system.security* priority: 501 @@ -8917,9 +9196,12 @@ elasticsearch: - logs-system.syslog@custom - so-fleet_globals-1 - so-fleet_agent_id_verification-1 + - so-system-mappings data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-system.syslog@custom index_patterns: - logs-system.syslog* priority: 501 @@ -8965,22 +9247,208 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-system.system@custom + index_patterns: + - logs-system.system* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-system.system-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-tenable_io_x_asset: + index_sorting: False + index_template: + index_patterns: + - "logs-tenable_io.asset-*" + template: + settings: + index: + lifecycle: + name: so-logs-tenable_io.asset-logs + number_of_replicas: 0 + composed_of: + - "logs-tenable_io.asset@package" + - "logs-tenable_io.asset@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + ignore_missing_component_templates: + - logs-tenable_io.asset@custom + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-tenable_io_x_plugin: + index_sorting: False + index_template: + index_patterns: + - "logs-tenable_io.plugin-*" + template: + settings: + index: + lifecycle: + name: so-logs-tenable_io.plugin-logs + number_of_replicas: 0 + composed_of: + - "logs-tenable_io.plugin@package" + - "logs-tenable_io.plugin@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + ignore_missing_component_templates: + - logs-tenable_io.plugin@custom + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-tenable_io_x_scan: + index_sorting: False + index_template: + index_patterns: + - "logs-tenable_io.scan-*" + template: + settings: + index: + lifecycle: + name: so-logs-tenable_io.scan-logs + number_of_replicas: 0 + composed_of: + - "logs-tenable_io.scan@package" + - "logs-tenable_io.scan@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + ignore_missing_component_templates: + - logs-tenable_io.scan@custom + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 30d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-tenable_io_x_vulnerability: + index_sorting: False + index_template: index_patterns: - - logs-system.system* - priority: 501 + - "logs-tenable_io.vulnerability-*" template: settings: index: lifecycle: - name: so-logs-system.system-logs + name: so-logs-tenable_io.vulnerability-logs number_of_replicas: 0 + composed_of: + - "logs-tenable_io.vulnerability@package" + - "logs-tenable_io.vulnerability@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + ignore_missing_component_templates: + - logs-tenable_io.vulnerability@custom policy: phases: cold: actions: set_priority: priority: 0 - min_age: 60d + min_age: 30d delete: actions: delete: {} @@ -9009,6 +9477,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-tenable_sc.asset@custom index_patterns: - logs-tenable_sc.asset-* priority: 501 @@ -9053,6 +9523,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-tenable_sc.plugin@custom index_patterns: - logs-tenable_sc.plugin-* priority: 501 @@ -9097,6 +9569,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-tenable_sc.vulnerability@custom index_patterns: - logs-tenable_sc.vulnerability-* priority: 501 @@ -9141,6 +9615,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_abusech.malware@custom index_patterns: - logs-ti_abusech.malware-* priority: 501 @@ -9185,6 +9661,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_abusech.malwarebazaar@custom index_patterns: - logs-ti_abusech.malwarebazaar-* priority: 501 @@ -9229,6 +9707,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_abusech.threatfox@custom index_patterns: - logs-ti_abusech.threatfox-* priority: 501 @@ -9273,6 +9753,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_abusech.url@custom index_patterns: - logs-ti_abusech.url-* priority: 501 @@ -9307,25 +9789,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-ti_anomali_x_threatstream: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-ti_anomali.threatstream@package + - logs-ti_anomali.threatstream@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-ti_anomali.threatstream@custom index_patterns: - - "logs-ti_anomali.threatstream-*" + - logs-ti_anomali.threatstream-* + priority: 501 template: settings: index: lifecycle: name: so-logs-ti_anomali.threatstream-logs number_of_replicas: 0 - composed_of: - - "logs-ti_anomali.threatstream@package" - - "logs-ti_anomali.threatstream@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -9351,25 +9835,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-ti_cybersixgill_x_threat: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-ti_cybersixgill.threat@package + - logs-ti_cybersixgill.threat@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-ti_cybersixgill.threat@custom index_patterns: - - "logs-ti_cybersixgill.threat-*" + - logs-ti_cybersixgill.threat-* + priority: 501 template: settings: index: lifecycle: name: so-logs-ti_cybersixgill.threat-logs number_of_replicas: 0 - composed_of: - - "logs-ti_cybersixgill.threat@package" - - "logs-ti_cybersixgill.threat@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -9405,6 +9891,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_misp.threat@custom index_patterns: - logs-ti_misp.threat-* priority: 501 @@ -9449,6 +9937,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_misp.threat_attributes@custom index_patterns: - logs-ti_misp.threat_attributes-* priority: 501 @@ -9493,6 +9983,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_otx.pulses_subscribed@custom index_patterns: - logs-ti_otx.pulses_subscribed-* priority: 501 @@ -9537,6 +10029,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_otx.threat@custom index_patterns: - logs-ti_otx.threat-* priority: 501 @@ -9581,6 +10075,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_recordedfuture.latest_ioc-template@custom index_patterns: - logs-ti_recordedfuture.latest_ioc-template-* priority: 501 @@ -9625,6 +10121,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_recordedfuture.threat@custom index_patterns: - logs-ti_recordedfuture.threat-* priority: 501 @@ -9659,25 +10157,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-ti_threatq_x_threat: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-ti_threatq.threat@package + - logs-ti_threatq.threat@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-ti_threatq.threat@custom index_patterns: - - "logs-ti_threatq.threat-*" + - logs-ti_threatq.threat-* + priority: 501 template: settings: index: lifecycle: name: so-logs-ti_threatq.threat-logs number_of_replicas: 0 - composed_of: - - "logs-ti_threatq.threat@package" - - "logs-ti_threatq.threat@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -9703,25 +10203,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-vsphere_x_log: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-vsphere.log@package + - logs-vsphere.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-vsphere.log@custom index_patterns: - - "logs-vsphere.log-*" + - logs-vsphere.log-* + priority: 501 template: settings: index: lifecycle: name: so-logs-vsphere.log-logs number_of_replicas: 0 - composed_of: - - "logs-vsphere.log@package" - - "logs-vsphere.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -9757,6 +10259,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-windows.forwarded@custom index_patterns: - logs-windows.forwarded* priority: 501 @@ -9801,6 +10305,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-windows.powershell@custom index_patterns: - logs-windows.powershell-* priority: 501 @@ -9845,6 +10351,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-windows.powershell_operational@custom index_patterns: - logs-windows.powershell_operational-* priority: 501 @@ -9889,6 +10397,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-windows.sysmon_operational@custom index_patterns: - logs-windows.sysmon_operational-* priority: 501 @@ -9923,25 +10433,28 @@ elasticsearch: priority: 50 min_age: 30d so-logs-winlog_x_winlog: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-winlog.winlog@package + - logs-winlog.winlog@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-winlog.winlog@package + - logs-winlog.winlog@custom index_patterns: - - "logs-winlog.winlog-*" + - logs-winlog.winlog-* + priority: 501 template: settings: index: lifecycle: name: so-logs-winlog.winlog-logs number_of_replicas: 0 - composed_of: - - "logs-winlog.winlog@package" - - "logs-winlog.winlog@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -9977,6 +10490,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zia.alerts@custom index_patterns: - logs-zscaler_zia.alerts-* priority: 501 @@ -10021,6 +10536,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zia.dns@custom index_patterns: - logs-zscaler_zia.dns-* priority: 501 @@ -10065,6 +10582,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zia.firewall@custom index_patterns: - logs-zscaler_zia.firewall-* priority: 501 @@ -10109,6 +10628,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zia.tunnel@custom index_patterns: - logs-zscaler_zia.tunnel-* priority: 501 @@ -10153,6 +10674,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zia.web@custom index_patterns: - logs-zscaler_zia.web-* priority: 501 @@ -10197,6 +10720,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zpa.app_connector_status@custom index_patterns: - logs-zscaler_zpa.app_connector_status-* priority: 501 @@ -10241,6 +10766,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zpa.audit@custom index_patterns: - logs-zscaler_zpa.audit-* priority: 501 @@ -10285,6 +10812,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zpa.browser_access@custom index_patterns: - logs-zscaler_zpa.browser_access-* priority: 501 @@ -10329,6 +10858,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zpa.user_activity@custom index_patterns: - logs-zscaler_zpa.user_activity-* priority: 501 @@ -10373,6 +10904,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zpa.user_status@custom index_patterns: - logs-zscaler_zpa.user_status-* priority: 501 @@ -10406,26 +10939,93 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-metrics-endpoint_x_metadata: - index_sorting: False + so-logstash: + index_sorting: false index_template: + composed_of: + - agent-mappings + - dtc-agent-mappings + - base-mappings + - dtc-base-mappings + - client-mappings + - dtc-client-mappings + - cloud-mappings + - container-mappings + - data_stream-mappings + - destination-mappings + - dtc-destination-mappings + - pb-override-destination-mappings + - dll-mappings + - dns-mappings + - dtc-dns-mappings + - ecs-mappings + - dtc-ecs-mappings + - error-mappings + - event-mappings + - dtc-event-mappings + - file-mappings + - dtc-file-mappings + - group-mappings + - host-mappings + - dtc-host-mappings + - http-mappings + - dtc-http-mappings + - log-mappings + - logstash-mappings + - network-mappings + - dtc-network-mappings + - observer-mappings + - dtc-observer-mappings + - orchestrator-mappings + - organization-mappings + - package-mappings + - process-mappings + - dtc-process-mappings + - registry-mappings + - related-mappings + - rule-mappings + - dtc-rule-mappings + - server-mappings + - service-mappings + - dtc-service-mappings + - source-mappings + - dtc-source-mappings + - pb-override-source-mappings + - threat-mappings + - tls-mappings + - tracing-mappings + - url-mappings + - user_agent-mappings + - dtc-user_agent-mappings + - vulnerability-mappings + - common-settings + - common-dynamic-mappings + ignore_missing_component_templates: [] index_patterns: - - "metrics-endpoint.metadata-*" + - logs-logstash-default* + priority: 500 template: + mappings: + date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string settings: index: lifecycle: - name: so-metrics-endpoint.metadata-logs + name: so-logstash-logs + mapping: + total_fields: + limit: 5000 number_of_replicas: 0 - composed_of: - - "metrics-endpoint.metadata@package" - - "metrics-endpoint.metadata@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc policy: phases: cold: @@ -10450,26 +11050,28 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-metrics-endpoint_x_metrics: - index_sorting: False + so-metrics-endpoint_x_metadata: + index_sorting: false index_template: + composed_of: + - metrics-endpoint.metadata@package + - metrics-endpoint.metadata@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics-endpoint.metadata@custom index_patterns: - - "metrics-endpoint.metrics-*" + - metrics-endpoint.metadata-* + priority: 501 template: settings: index: lifecycle: - name: so-metrics-endpoint.metrics-logs + name: so-metrics-endpoint.metadata-logs number_of_replicas: 0 - composed_of: - - "metrics-endpoint.metrics@package" - - "metrics-endpoint.metrics@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -10494,26 +11096,28 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-metrics-endpoint_x_policy: - index_sorting: False + so-metrics-endpoint_x_metrics: + index_sorting: false index_template: + composed_of: + - metrics-endpoint.metrics@package + - metrics-endpoint.metrics@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics-endpoint.metrics@custom index_patterns: - - "metrics-endpoint.policy-*" + - metrics-endpoint.metrics-* + priority: 501 template: settings: index: lifecycle: - name: so-metrics-endpoint.policy-logs + name: so-metrics-endpoint.metrics-logs number_of_replicas: 0 - composed_of: - - "metrics-endpoint.policy@package" - - "metrics-endpoint.policy@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -10538,26 +11142,28 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-metrics-vsphere_x_datastore: - index_sorting: False + so-metrics-endpoint_x_policy: + index_sorting: false index_template: + composed_of: + - metrics-endpoint.policy@package + - metrics-endpoint.policy@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics-endpoint.policy@custom index_patterns: - - "metrics-vsphere.datastore-*" + - metrics-endpoint.policy-* + priority: 501 template: settings: index: lifecycle: - name: so-metrics-vsphere.datastore-logs + name: so-metrics-endpoint.policy-logs number_of_replicas: 0 - composed_of: - - "metrics-vsphere.datastore@package" - - "metrics-vsphere.datastore@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -10582,26 +11188,74 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-metrics-vsphere_x_host: - index_sorting: False + so-metrics-fleet_server_x_agent_status: + index_sorting: false index_template: + composed_of: + - metrics@tsdb-settings + - metrics-fleet_server.agent_status@package + - metrics-fleet_server.agent_status@custom + - ecs@mappings + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics-fleet_server.agent_status@custom index_patterns: - - "metrics-vsphere.host-*" + - metrics-fleet_server.agent_status-* + priority: 501 template: settings: index: - lifecycle: - name: so-metrics-vsphere.host-logs + mode: time_series number_of_replicas: 0 + so-metrics-fleet_server_x_agent_versions: + index_sorting: false + index_template: composed_of: - - "metrics-vsphere.host@package" - - "metrics-vsphere.host@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 + - metrics@tsdb-settings + - metrics-fleet_server.agent_versions@package + - metrics-fleet_server.agent_versions@custom + - ecs@mappings + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 data_stream: + allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - metrics-fleet_server.agent_versions@custom + index_patterns: + - metrics-fleet_server.agent_versions-* + priority: 501 + template: + settings: + index: + mode: time_series + number_of_replicas: 0 + so-metrics-nginx_x_stubstatus: + index_sorting: false + index_template: + composed_of: + - metrics-nginx.stubstatus@package + - metrics-nginx.stubstatus@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics-nginx.stubstatus@custom + index_patterns: + - metrics-nginx.stubstatus-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-nginx.stubstatus-logs + number_of_replicas: 0 policy: phases: cold: @@ -10626,26 +11280,28 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-metrics-vsphere_x_virtualmachine: - index_sorting: False + so-metrics-vsphere_x_datastore: + index_sorting: false index_template: + composed_of: + - metrics-vsphere.datastore@package + - metrics-vsphere.datastore@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics-vsphere.datastore@custom index_patterns: - - "metrics-vsphere.virtualmachine-*" + - metrics-vsphere.datastore-* + priority: 501 template: settings: index: lifecycle: - name: so-metrics-vsphere.virtualmachine-logs + name: so-metrics-vsphere.datastore-logs number_of_replicas: 0 - composed_of: - - "metrics-vsphere.virtualmachine@package" - - "metrics-vsphere.virtualmachine@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -10670,28 +11326,27 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-kismet: + so-metrics-vsphere_x_host: index_sorting: false index_template: composed_of: - - kismet-mappings - - source-mappings - - client-mappings - - device-mappings - - network-mappings + - metrics-vsphere.host@package + - metrics-vsphere.host@custom - so-fleet_globals-1 - so-fleet_agent_id_verification-1 data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - metrics-vsphere.host@custom index_patterns: - - logs-kismet-so* + - metrics-vsphere.host-* priority: 501 template: settings: index: lifecycle: - name: so-kismet-logs + name: so-metrics-vsphere.host-logs number_of_replicas: 0 policy: phases: @@ -10717,92 +11372,28 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-logstash: + so-metrics-vsphere_x_virtualmachine: index_sorting: false index_template: composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - cloud-mappings - - container-mappings - - data_stream-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - logstash-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - orchestrator-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - registry-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - tracing-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - vulnerability-mappings - - common-settings - - common-dynamic-mappings + - metrics-vsphere.virtualmachine@package + - metrics-vsphere.virtualmachine@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics-vsphere.virtualmachine@custom index_patterns: - - logs-logstash-default* - priority: 500 + - metrics-vsphere.virtualmachine-* + priority: 501 template: - mappings: - date_detection: false - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string settings: index: lifecycle: - name: so-logstash-logs - mapping: - total_fields: - limit: 5000 + name: so-metrics-vsphere.virtualmachine-logs number_of_replicas: 0 - number_of_shards: 1 - refresh_interval: 30s - sort: - field: '@timestamp' - order: desc policy: phases: cold: @@ -10888,6 +11479,7 @@ elasticsearch: - vulnerability-mappings - common-settings - common-dynamic-mappings + ignore_missing_component_templates: [] index_patterns: - logs-redis-default* priority: 500 @@ -11000,6 +11592,7 @@ elasticsearch: - common-settings - common-dynamic-mappings data_stream: {} + ignore_missing_component_templates: [] index_patterns: - logs-strelka-so* priority: 500 @@ -11111,6 +11704,7 @@ elasticsearch: - common-settings - common-dynamic-mappings data_stream: {} + ignore_missing_component_templates: [] index_patterns: - logs-suricata-so* priority: 500 @@ -11222,6 +11816,7 @@ elasticsearch: - common-settings - common-dynamic-mappings data_stream: {} + ignore_missing_component_templates: [] index_patterns: - logs-suricata.alerts-* priority: 500 @@ -11334,6 +11929,7 @@ elasticsearch: - common-settings - common-dynamic-mappings data_stream: {} + ignore_missing_component_templates: [] index_patterns: - logs-syslog-so* priority: 500 @@ -11447,6 +12043,7 @@ elasticsearch: - common-settings - common-dynamic-mappings data_stream: {} + ignore_missing_component_templates: [] index_patterns: - logs-zeek-so* priority: 500 @@ -11496,6 +12093,87 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + pipelines: + custom001: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom001 + - pipeline: + name: common + custom002: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom002 + - pipeline: + name: common + custom003: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom003 + - pipeline: + name: common + custom004: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom004 + - pipeline: + name: common + custom005: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom005 + - pipeline: + name: common + custom006: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom006 + - pipeline: + name: common + custom007: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom007 + - pipeline: + name: common + custom008: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom008 + - pipeline: + name: common + custom009: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom009 + - pipeline: + name: common + custom010: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom010 + - pipeline: + name: common retention: retention_pct: 50 so_roles: diff --git a/salt/elasticsearch/files/ingest-dynamic/common b/salt/elasticsearch/files/ingest-dynamic/common index 836b8d4af0..e847029092 100644 --- a/salt/elasticsearch/files/ingest-dynamic/common +++ b/salt/elasticsearch/files/ingest-dynamic/common @@ -62,6 +62,7 @@ { "split": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "dataset_tag_temp" } }, { "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp.1}}" } }, { "grok": { "if": "ctx.http?.response?.status_code != null", "field": "http.response.status_code", "patterns": ["%{NUMBER:http.response.status_code:long} %{GREEDYDATA}"]} }, + { "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } }, { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "dataset_tag_temp", "event.dataset_temp" ], "ignore_missing": true, "ignore_failure": true } } {%- endraw %} {%- if HIGHLANDER %} @@ -72,7 +73,9 @@ } } {%- endif %} -{%- raw %} +{%- raw %} + , + { "pipeline": { "name": "global@custom", "ignore_missing_pipeline": true, "description": "[Fleet] Global pipeline for all data streams" } } ] } {% endraw %} diff --git a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 b/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 deleted file mode 100644 index 233cd647b4..0000000000 --- a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 +++ /dev/null @@ -1,107 +0,0 @@ -{ - "version": 3, - "_meta": { - "managed_by": "fleet", - "managed": true - }, - "description": "Final pipeline for processing all incoming Fleet Agent documents. \n", - "processors": [ - { - "date": { - "description": "Add time when event was ingested (and remove sub-seconds to improve storage efficiency)", - "tag": "truncate-subseconds-event-ingested", - "field": "_ingest.timestamp", - "target_field": "event.ingested", - "formats": [ - "ISO8601" - ], - "output_format": "date_time_no_millis", - "ignore_failure": true - } - }, - { - "remove": { - "description": "Remove any pre-existing untrusted values.", - "field": [ - "event.agent_id_status", - "_security" - ], - "ignore_missing": true - } - }, - { - "set_security_user": { - "field": "_security", - "properties": [ - "authentication_type", - "username", - "realm", - "api_key" - ] - } - }, - { - "script": { - "description": "Add event.agent_id_status based on the API key metadata and the agent.id contained in the event.\n", - "tag": "agent-id-status", - "source": "boolean is_user_trusted(def ctx, def users) {\n if (ctx?._security?.username == null) {\n return false;\n }\n\n def user = null;\n for (def item : users) {\n if (item?.username == ctx._security.username) {\n user = item;\n break;\n }\n }\n\n if (user == null || user?.realm == null || ctx?._security?.realm?.name == null) {\n return false;\n }\n\n if (ctx._security.realm.name != user.realm) {\n return false;\n }\n\n return true;\n}\n\nString verified(def ctx, def params) {\n // No agent.id field to validate.\n if (ctx?.agent?.id == null) {\n return \"missing\";\n }\n\n // Check auth metadata from API key.\n if (ctx?._security?.authentication_type == null\n // Agents only use API keys.\n || ctx._security.authentication_type != 'API_KEY'\n // Verify the API key owner before trusting any metadata it contains.\n || !is_user_trusted(ctx, params.trusted_users)\n // Verify the API key has metadata indicating the assigned agent ID.\n || ctx?._security?.api_key?.metadata?.agent_id == null) {\n return \"auth_metadata_missing\";\n }\n\n // The API key can only be used represent the agent.id it was issued to.\n if (ctx._security.api_key.metadata.agent_id != ctx.agent.id) {\n // Potential masquerade attempt.\n return \"mismatch\";\n }\n\n return \"verified\";\n}\n\nif (ctx?.event == null) {\n ctx.event = [:];\n}\n\nctx.event.agent_id_status = verified(ctx, params);", - "params": { - "trusted_users": [ - { - "username": "elastic/fleet-server", - "realm": "_service_account" - }, - { - "username": "cloud-internal-agent-server", - "realm": "found" - }, - { - "username": "elastic", - "realm": "reserved" - } - ] - } - } - }, - { - "remove": { - "field": "_security", - "ignore_missing": true - } - }, - { "set": { "ignore_failure": true, "field": "event.module", "value": "elastic_agent" } }, - { "split": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp" } }, - { "set": { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } }, - { "gsub": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } }, - { "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}" } }, - { "set": { "if": "ctx.network?.direction == 'egress'", "override": true, "field": "network.initiated", "value": "true" } }, - { "set": { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } }, - { "set": { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } }, - { "set": { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } }, - { "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.dataset", "value": "import" } }, - { "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.namespace", "value": "so" } }, - { "date": { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } }, - { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true } }, - { "set": { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } }, - { "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } }, - { "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } }, - { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } } - ], - "on_failure": [ - { - "remove": { - "field": "_security", - "ignore_missing": true, - "ignore_failure": true - } - }, - { - "append": { - "field": "error.message", - "value": [ - "failed in Fleet agent final_pipeline: {{ _ingest.on_failure_message }}" - ] - } - } - ] -} diff --git a/salt/elasticsearch/files/ingest/global@custom b/salt/elasticsearch/files/ingest/global@custom new file mode 100644 index 0000000000..dbf215fb1b --- /dev/null +++ b/salt/elasticsearch/files/ingest/global@custom @@ -0,0 +1,27 @@ +{ + "version": 3, + "_meta": { + "managed_by": "securityonion", + "managed": true + }, + "description": "Custom pipeline for processing all incoming Fleet Agent documents. \n", + "processors": [ + { "set": { "ignore_failure": true, "field": "event.module", "value": "elastic_agent" } }, + { "split": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp" } }, + { "set": { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } }, + { "gsub": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } }, + { "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}" } }, + { "set": { "if": "ctx.network?.direction == 'egress'", "override": true, "field": "network.initiated", "value": "true" } }, + { "set": { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } }, + { "set": { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } }, + { "set": { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } }, + { "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.dataset", "value": "import" } }, + { "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.namespace", "value": "so" } }, + { "date": { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } }, + { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true } }, + { "set": { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } }, + { "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } }, + { "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } }, + { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } } + ] +} diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index f56ed313e3..d30706837d 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -466,6 +466,13 @@ elasticsearch: so-logs-sonicwall_firewall_x_log: *indexSettings so-logs-snort_x_log: *indexSettings so-logs-symantec_endpoint_x_log: *indexSettings + so-logs-tenable_io_x_asset: *indexSettings + so-logs-tenable_io_x_plugin: *indexSettings + so-logs-tenable_io_x_scan: *indexSettings + so-logs-tenable_io_x_vulnerability: *indexSettings + so-logs-tenable_sc_x_asset: *indexSettings + so-logs-tenable_sc_x_plugin: *indexSettings + so-logs-tenable_sc_x_vulnerability: *indexSettings so-logs-ti_abusech_x_malware: *indexSettings so-logs-ti_abusech_x_malwarebazaar: *indexSettings so-logs-ti_abusech_x_threatfox: *indexSettings @@ -530,6 +537,58 @@ elasticsearch: so-strelka: *indexSettings so-syslog: *indexSettings so-zeek: *indexSettings + so-metrics-fleet_server_x_agent_status: &fleetMetricsSettings + index_sorting: + description: Sorts the index by event time, at the cost of additional processing resource consumption. + advanced: True + readonly: True + helpLink: elasticsearch.html + index_template: + ignore_missing_component_templates: + description: Ignore component templates if they aren't in Elasticsearch. + advanced: True + readonly: True + helpLink: elasticsearch.html + index_patterns: + description: Patterns for matching multiple indices or tables. + advanced: True + readonly: True + helpLink: elasticsearch.html + template: + settings: + index: + mode: + description: Type of mode used for this index. Time series indices can be used for metrics to reduce necessary storage. + advanced: True + readonly: True + helpLink: elasticsearch.html + number_of_replicas: + description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs. + advanced: True + readonly: True + helpLink: elasticsearch.html + composed_of: + description: The index template is composed of these component templates. + advanced: True + readonly: True + helpLink: elasticsearch.html + priority: + description: The priority of the index template. + advanced: True + readonly: True + helpLink: elasticsearch.html + data_stream: + hidden: + description: Hide the data stream. + advanced: True + readonly: True + helpLink: elasticsearch.html + allow_custom_routing: + description: Allow custom routing for the data stream. + advanced: True + readonly: True + helpLink: elasticsearch.html + so-metrics-fleet_server_x_agent_versions: *fleetMetricsSettings so_roles: so-manager: &soroleSettings config: diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json index 7bcc34de93..efd85bb4b4 100644 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json +++ b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json @@ -6,7 +6,7 @@ "name": "logs" }, "codec": "best_compression", - "default_pipeline": "logs-elastic_agent-1.13.1", + "default_pipeline": "logs-elastic_agent-1.20.0", "mapping": { "total_fields": { "limit": "10000" diff --git a/salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_status@package.json b/salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_status@package.json new file mode 100644 index 0000000000..8fc83f9cb8 --- /dev/null +++ b/salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_status@package.json @@ -0,0 +1,201 @@ +{ + "template": { + "settings": { + "index": { + "lifecycle": { + "name": "metrics" + }, + "default_pipeline": "metrics-fleet_server.agent_status-1.5.0", + "mapping": { + "total_fields": { + "limit": "1000" + } + } + } + }, + "mappings": { + "dynamic": false, + "_source": { + "mode": "synthetic" + }, + "properties": { + "cluster": { + "properties": { + "id": { + "time_series_dimension": true, + "type": "keyword" + } + } + }, + "fleet": { + "properties": { + "agents": { + "properties": { + "offline": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "total": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "updating": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "inactive": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "healthy": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "unhealthy": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "unenrolled": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "enrolled": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "unhealthy_reason": { + "properties": { + "output": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "input": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "other": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + } + } + }, + "upgrading_step": { + "properties": { + "rollback": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "requested": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "restarting": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "downloading": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "scheduled": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "extracting": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "replacing": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "failed": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "watching": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + } + } + } + } + } + } + }, + "agent": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "@timestamp": { + "ignore_malformed": false, + "type": "date" + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + }, + "type": { + "type": "constant_keyword" + }, + "dataset": { + "type": "constant_keyword" + } + } + }, + "kibana": { + "properties": { + "uuid": { + "path": "agent.id", + "type": "alias" + }, + "version": { + "path": "agent.version", + "type": "alias" + } + } + } + } + } + }, + "_meta": { + "package": { + "name": "fleet_server" + }, + "managed_by": "fleet", + "managed": true + } +} diff --git a/salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_versions@package.json b/salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_versions@package.json new file mode 100644 index 0000000000..af3323ee90 --- /dev/null +++ b/salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_versions@package.json @@ -0,0 +1,102 @@ +{ + "template": { + "settings": { + "index": { + "lifecycle": { + "name": "metrics" + }, + "default_pipeline": "metrics-fleet_server.agent_versions-1.5.0", + "mapping": { + "total_fields": { + "limit": "1000" + } + } + } + }, + "mappings": { + "dynamic": false, + "_source": { + "mode": "synthetic" + }, + "properties": { + "cluster": { + "properties": { + "id": { + "time_series_dimension": true, + "type": "keyword" + } + } + }, + "fleet": { + "properties": { + "agent": { + "properties": { + "count": { + "time_series_metric": "gauge", + "meta": {}, + "type": "long" + }, + "version": { + "time_series_dimension": true, + "type": "keyword" + } + } + } + } + }, + "agent": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "@timestamp": { + "ignore_malformed": false, + "type": "date" + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + }, + "type": { + "type": "constant_keyword" + }, + "dataset": { + "type": "constant_keyword" + } + } + }, + "kibana": { + "properties": { + "uuid": { + "path": "agent.id", + "type": "alias" + }, + "version": { + "path": "agent.version", + "type": "alias" + } + } + } + } + } + }, + "_meta": { + "package": { + "name": "fleet_server" + }, + "managed_by": "fleet", + "managed": true + } +} diff --git a/salt/elasticsearch/templates/component/so/so-system-mappings.json b/salt/elasticsearch/templates/component/so/so-system-mappings.json new file mode 100644 index 0000000000..f86c427a6e --- /dev/null +++ b/salt/elasticsearch/templates/component/so/so-system-mappings.json @@ -0,0 +1,29 @@ +{ + "template": { + "mappings": { + "properties": { + "host": { + "properties":{ + "ip": { + "type": "ip" + } + } + }, + "related": { + "properties":{ + "ip": { + "type": "ip" + } + } + }, + "source": { + "properties":{ + "ip": { + "type": "ip" + } + } + } + } + } + } +} diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines index 71c40c1ca2..b76a0e0f01 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines @@ -20,7 +20,7 @@ if [ ! -f /opt/so/state/espipelines.txt ]; then cd ${ELASTICSEARCH_INGEST_PIPELINES} echo "Loading pipelines..." - for i in .[a-z]* *; + for i in *; do echo $i; retry 5 5 "so-elasticsearch-query _ingest/pipeline/$i -d@$i -XPUT | grep '{\"acknowledged\":true}'" || fail "Could not load pipeline: $i" diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-total b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-total index b73967c26f..e507333499 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-total +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-total @@ -40,9 +40,9 @@ fi # Iterate through the output of _cat/allocation for each node in the cluster to determine the total available space {% if GLOBALS.role == 'so-manager' %} -for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | grep -v "{{ GLOBALS.manager }}$" | awk '{print $5}'); do +for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | grep -v "{{ GLOBALS.manager }}$" | awk '{print $8}'); do {% else %} -for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | awk '{print $5}'); do +for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | awk '{print $8}'); do {% endif %} size=$(echo $i | grep -oE '[0-9].*' | awk '{print int($1+0.5)}') unit=$(echo $i | grep -oE '[A-Za-z]+') diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-used b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-used index 49e6348531..b18e84d719 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-used +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-used @@ -13,10 +13,10 @@ TOTAL_USED_SPACE=0 # Iterate through the output of _cat/allocation for each node in the cluster to determine the total used space {% if GLOBALS.role == 'so-manager' %} # Get total disk space - disk.total -for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | grep -v "{{ GLOBALS.manager }}$" | awk '{print $3}'); do +for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | grep -v "{{ GLOBALS.manager }}$" | awk '{print $6}'); do {% else %} # Get disk space taken up by indices - disk.indices -for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | awk '{print $2}'); do +for i in $(/usr/sbin/so-elasticsearch-query _cat/allocation | awk '{print $5}'); do {% endif %} size=$(echo $i | grep -oE '[0-9].*' | awk '{print int($1+0.5)}') unit=$(echo $i | grep -oE '[A-Za-z]+') diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load index b00fcbedfd..04a7a8ab0b 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load @@ -10,10 +10,26 @@ {%- for index, settings in ES_INDEX_SETTINGS.items() %} {%- if settings.policy is defined %} -echo -echo "Setting up {{ index }}-logs policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' -echo +{%- if index == 'so-logs-detections.alerts' %} + echo + echo "Setting up so-logs-detections.alerts-so policy..." + curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-so" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' + echo +{%- elif index == 'so-logs-soc' %} + echo + echo "Setting up so-soc-logs policy..." + curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/so-soc-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' + echo + echo + echo "Setting up {{ index }}-logs policy..." + curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' + echo +{%- else %} + echo + echo "Setting up {{ index }}-logs policy..." + curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' + echo +{%- endif %} {%- endif %} {%- endfor %} echo diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load index 080348522f..76b1cc193f 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -5,7 +5,6 @@ # Elastic License 2.0. {%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} {% from 'vars/globals.map.jinja' import GLOBALS %} -{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %} STATE_FILE_INITIAL=/opt/so/state/estemplates_initial_load_attempt.txt STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt @@ -68,9 +67,9 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then echo -n "Waiting for ElasticSearch..." retry 240 1 "so-elasticsearch-query / -k --output /dev/null --silent --head --fail" || fail "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" {% if GLOBALS.role != 'so-heavynode' %} - SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') - INSTALLED=$(elastic_fleet_package_is_installed {{ SUPPORTED_PACKAGES[0] }} ) - if [ "$INSTALLED" != "installed" ]; then + TEMPLATE="logs-endpoint.alerts@package" + INSTALLED=$(so-elasticsearch-query _component_template/$TEMPLATE | jq -r .component_templates[0].name) + if [ "$INSTALLED" != "$TEMPLATE" ]; then echo echo "Packages not yet installed." echo @@ -134,7 +133,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then TEMPLATE=${i::-14} COMPONENT_PATTERN=${TEMPLATE:3} MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery") - if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" ]]; then + if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ logs-http_endpoint\.generic|logs-winlog\.winlog ]]; then load_failures=$((load_failures+1)) echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures" else @@ -153,7 +152,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then cd - >/dev/null if [[ $load_failures -eq 0 ]]; then - echo "All template loaded successfully" + echo "All templates loaded successfully" touch $STATE_FILE_SUCCESS else echo "Encountered $load_failures templates that were unable to load, likely due to missing dependencies that will be available later; will retry on next highstate" diff --git a/salt/firewall/soc_firewall.yaml b/salt/firewall/soc_firewall.yaml index 91aec0b0dc..3a8b4d3a04 100644 --- a/salt/firewall/soc_firewall.yaml +++ b/salt/firewall/soc_firewall.yaml @@ -120,7 +120,10 @@ firewall: influxdb: tcp: *tcpsettings udp: *udpsettings - kafka: + kafka_controller: + tcp: *tcpsettings + udp: *udpsettings + kafka_data: tcp: *tcpsettings udp: *udpsettings kibana: diff --git a/salt/idstools/etc/rulecat.conf b/salt/idstools/etc/rulecat.conf index db78cec29c..e4ec611dbc 100644 --- a/salt/idstools/etc/rulecat.conf +++ b/salt/idstools/etc/rulecat.conf @@ -1,6 +1,6 @@ {%- from 'vars/globals.map.jinja' import GLOBALS -%} {%- from 'soc/merged.map.jinja' import SOCMERGED -%} ---suricata-version=6.0 +--suricata-version=7.0.3 --merged=/opt/so/rules/nids/suri/all.rules --output=/nsm/rules/detect-suricata/custom_temp --local=/opt/so/rules/nids/suri/local.rules @@ -20,4 +20,4 @@ --local={{ ruleset.file }} {%- endif %} {%- endfor %} -{%- endif %} \ No newline at end of file +{%- endif %} diff --git a/salt/idstools/tools/sbin_jinja/so-rule-update b/salt/idstools/tools/sbin_jinja/so-rule-update index 4ea79c94ed..9ac09ed15c 100755 --- a/salt/idstools/tools/sbin_jinja/so-rule-update +++ b/salt/idstools/tools/sbin_jinja/so-rule-update @@ -23,9 +23,9 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then {%- if not GLOBALS.airgap %} # Download the rules from the internet {%- if IDSTOOLSMERGED.config.ruleset == 'ETOPEN' %} - docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force + docker exec so-idstools idstools-rulecat -v --suricata-version 7.0.3 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force {%- elif IDSTOOLSMERGED.config.ruleset == 'ETPRO' %} - docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --etpro={{ IDSTOOLSMERGED.config.oinkcode }} + docker exec so-idstools idstools-rulecat -v --suricata-version 7.0.3 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --etpro={{ IDSTOOLSMERGED.config.oinkcode }} {%- endif %} {%- endif %} diff --git a/salt/kibana/files/config_saved_objects.ndjson b/salt/kibana/files/config_saved_objects.ndjson index bc503debb0..c52b5cb44b 100644 --- a/salt/kibana/files/config_saved_objects.ndjson +++ b/salt/kibana/files/config_saved_objects.ndjson @@ -1 +1,2 @@ -{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.10.4","id": "8.10.4","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} +{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.14.3","id": "8.14.3","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} + diff --git a/salt/kibana/tools/sbin_jinja/so-kibana-config-load b/salt/kibana/tools/sbin_jinja/so-kibana-config-load index 5b4b525602..8177adb5c2 100644 --- a/salt/kibana/tools/sbin_jinja/so-kibana-config-load +++ b/salt/kibana/tools/sbin_jinja/so-kibana-config-load @@ -63,7 +63,7 @@ update() { IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' for i in "${LINES[@]}"; do - RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.10.4" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") + RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.14.3" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi done diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 6f14104c3f..ebbfa8fff3 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -9,6 +9,10 @@ if [ -f /usr/sbin/so-common ]; then . /usr/sbin/so-common fi +if [ -f /usr/sbin/so-elastic-fleet-common ]; then + . /usr/sbin/so-elastic-fleet-common +fi + function usage() { echo "Usage: $0 -o= -m=[id]" echo "" @@ -380,23 +384,31 @@ function add_elastic_fleet_package_registry_to_minion() { function create_fleet_policy() { - JSON_STRING=$( jq -n \ - --arg NAME "FleetServer_$LSHOSTNAME" \ - --arg DESC "Fleet Server - $LSHOSTNAME" \ - '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":1209600,"has_fleet_server":true}' - ) + # First, set the default output to Elasticsearch + # This is required because of the license output bug + JSON_STRING=$(jq -n \ + '{ + "name": "so-manager_elasticsearch", + "type": "elasticsearch", + "is_default": true, + "is_default_monitoring": false + }') + + curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" + + # Create the Fleet Server Policy + elastic_fleet_policy_create "FleetServer_$LSHOSTNAME" "Fleet Server - $LSHOSTNAME" "false" "120" - # Create Fleet Sever Policy - curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" + # Modify the default integration policy to update the policy_id with the correct naming + UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "FleetServer_$LSHOSTNAME" --arg name "fleet_server-$LSHOSTNAME" ' + .policy_id = $policy_id | + .name = $name' /opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json) - JSON_STRING_UPDATE=$( jq -n \ - --arg NAME "FleetServer_$LSHOSTNAME" \ - --arg DESC "Fleet Server - $LSHOSTNAME" \ - '{"name":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":120,"data_output_id":"so-manager_elasticsearch"}' - ) + # Add the Fleet Server Integration to the new Fleet Policy + elastic_fleet_integration_create "$UPDATED_INTEGRATION_POLICY" - # Update Fleet Policy - ES Output - curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/FleetServer_$LSHOSTNAME" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING_UPDATE" + # Set the default output back to the default + /sbin/so-elastic-fleet-outputs-update } function update_fleet_host_urls() { diff --git a/salt/manager/tools/sbin/so-user b/salt/manager/tools/sbin/so-user index 69b4fdb508..40e0b01f8d 100755 --- a/salt/manager/tools/sbin/so-user +++ b/salt/manager/tools/sbin/so-user @@ -234,10 +234,14 @@ function updatePassword() { passwordHash=$(hashPassword "$password") # Update DB with new hash echo "update identity_credentials set config=CAST('{\"hashed_password\":\"$passwordHash\"}' as BLOB), created_at=datetime('now'), updated_at=datetime('now') where identity_id='${identityId}' and identity_credential_type_id=(select id from identity_credential_types where name='password');" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath" - # Deactivate MFA - echo "delete from identity_credential_identifiers where identity_credential_id=(select id from identity_credentials where identity_id='${identityId}' and identity_credential_type_id=(select id from identity_credential_types where name in ('totp', 'webauthn', 'oidc')));" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath" - echo "delete from identity_credentials where identity_id='${identityId}' and identity_credential_type_id=(select id from identity_credential_types where name in ('totp', 'webauthn', 'oidc'));" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath" [[ $? != 0 ]] && fail "Unable to update password" + # Deactivate MFA + echo "delete from identity_credential_identifiers where identity_credential_id in (select id from identity_credentials where identity_id='${identityId}' and identity_credential_type_id in (select id from identity_credential_types where name in ('totp', 'webauthn', 'oidc')));" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath" + [[ $? != 0 ]] && fail "Unable to clear aal2 identity IDs" + echo "delete from identity_credentials where identity_id='${identityId}' and identity_credential_type_id in (select id from identity_credential_types where name in ('totp', 'webauthn', 'oidc'));" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath" + [[ $? != 0 ]] && fail "Unable to clear aal2 identity credentials" + echo "update identities set available_aal='aal1' where id='${identityId}';" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath" + [[ $? != 0 ]] && fail "Unable to reset aal" fi } diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index bc13066738..c74edf1ebe 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -401,6 +401,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.4.60 ]] && up_to_2.4.70 [[ "$INSTALLEDVERSION" == 2.4.70 ]] && up_to_2.4.80 [[ "$INSTALLEDVERSION" == 2.4.80 ]] && up_to_2.4.90 + [[ "$INSTALLEDVERSION" == 2.4.90 ]] && up_to_2.4.100 true } @@ -420,6 +421,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == 2.4.60 ]] && post_to_2.4.70 [[ "$POSTVERSION" == 2.4.70 ]] && post_to_2.4.80 [[ "$POSTVERSION" == 2.4.80 ]] && post_to_2.4.90 + [[ "$POSTVERSION" == 2.4.90 ]] && post_to_2.4.100 true } @@ -451,8 +453,6 @@ post_to_2.4.20() { } post_to_2.4.30() { - echo "Regenerating Elastic Agent Installers" - /sbin/so-elastic-agent-gen-installers # there is an occasional error with this state: pki_public_ca_crt: TypeError: list indices must be integers or slices, not str set +e salt-call state.apply ca queue=True @@ -477,8 +477,7 @@ post_to_2.4.50() { } post_to_2.4.60() { - echo "Regenerating Elastic Agent Installers..." - so-elastic-agent-gen-installers + echo "Nothing to apply" POSTVERSION=2.4.60 } @@ -504,6 +503,12 @@ post_to_2.4.90() { POSTVERSION=2.4.90 } +post_to_2.4.100() { + echo "Regenerating Elastic Agent Installers" + /sbin/so-elastic-agent-gen-installers + POSTVERSION=2.4.100 +} + repo_sync() { echo "Sync the local repo." su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync." @@ -580,18 +585,7 @@ up_to_2.4.20() { } up_to_2.4.30() { - - # Remove older defend integration json & installed integration - rm -f /opt/so/conf/elastic-fleet/integrations/endpoints-initial/elastic-defend-endpoints.json - - . $UPDATE_DIR/salt/elasticfleet/tools/sbin/so-elastic-fleet-common - elastic_fleet_integration_remove endpoints-initial elastic-defend-endpoints - - rm -f /opt/so/state/eaintegrations.txt - - # Elastic Update for this release, so download Elastic Agent files - determine_elastic_agent_upgrade - rm -f /opt/so/state/estemplates*.txt + echo "Nothing to do for 2.4.30" INSTALLEDVERSION=2.4.30 } @@ -681,10 +675,16 @@ up_to_2.4.90() { so-yaml.py remove /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.password so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.password "$kafkatrimpass" so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.trustpass "$kafkatrust" + echo "If the Detection index exists, update the refresh_interval" so-elasticsearch-query so-detection*/_settings -X PUT -d '{"index":{"refresh_interval":"1s"}}' INSTALLEDVERSION=2.4.90 } +up_to_2.4.100() { + # Elastic Update for this release, so download Elastic Agent files + determine_elastic_agent_upgrade + INSTALLEDVERSION=2.4.100 +} add_detection_test_pillars() { if [[ -n "$SOUP_INTERNAL_TESTING" ]]; then @@ -945,7 +945,9 @@ upgrade_salt() { if [[ $is_rpm ]]; then echo "Removing yum versionlock for Salt." echo "" - yum versionlock delete "salt-*" + yum versionlock delete "salt" + yum versionlock delete "salt-minion" + yum versionlock delete "salt-master" echo "Updating Salt packages." echo "" set +e @@ -963,7 +965,9 @@ upgrade_salt() { set -e echo "Applying yum versionlock for Salt." echo "" - yum versionlock add "salt-*" + yum versionlock add "salt-0:$NEWSALTVERSION-0.*" + yum versionlock add "salt-minion-0:$NEWSALTVERSION-0.*" + yum versionlock add "salt-master-0:$NEWSALTVERSION-0.*" # Else do Ubuntu things elif [[ $is_deb ]]; then echo "Removing apt hold for Salt." @@ -1404,6 +1408,8 @@ Please review the following for more information about the update process and re $DOC_BASE_URL/soup.html https://blog.securityonion.net +WARNING: If you run soup via an SSH session and that SSH session terminates, then any processes running in that session would terminate. You should avoid leaving soup unattended especially if the machine you are SSHing from is configured to sleep after a period of time. You might also consider using something like screen or tmux so that if your SSH session terminates, the processes will continue running on the server. + EOF cat << EOF diff --git a/salt/registry/enabled.sls b/salt/registry/enabled.sls index 9ea3ad1df6..bff46dd5ba 100644 --- a/salt/registry/enabled.sls +++ b/salt/registry/enabled.sls @@ -14,7 +14,7 @@ include: # Install the registry container so-dockerregistry: docker_container.running: - - image: ghcr.io/security-onion-solutions/registry:2.8.2 + - image: ghcr.io/security-onion-solutions/registry:2.8.3 - hostname: so-registry - networks: - sobridge: diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 5f687ef3f3..cc388edb43 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -3,12 +3,10 @@ {% if grains.os_family == 'Debian' %} {% set SPLITCHAR = '+' %} - {% set SALTNOTHELD = salt['cmd.run']('apt-mark showhold | grep -q salt ; echo $?', python_shell=True) %} {% set SALTPACKAGES = ['salt-common', 'salt-master', 'salt-minion'] %} {% set SYSTEMD_UNIT_FILE = '/lib/systemd/system/salt-minion.service' %} {% else %} {% set SPLITCHAR = '-' %} - {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep -q salt ; echo $?', python_shell=True) %} {% set SALTPACKAGES = ['salt', 'salt-master', 'salt-minion'] %} {% set SYSTEMD_UNIT_FILE = '/usr/lib/systemd/system/salt-minion.service' %} {% endif %} diff --git a/salt/salt/master.defaults.yaml b/salt/salt/master.defaults.yaml index 19677f70bd..e133dbd0b7 100644 --- a/salt/salt/master.defaults.yaml +++ b/salt/salt/master.defaults.yaml @@ -1,4 +1,4 @@ # version cannot be used elsewhere in this pillar as soup is grepping for it to determine if Salt needs to be patched salt: master: - version: 3006.6 + version: 3006.9 diff --git a/salt/salt/master.sls b/salt/salt/master.sls index 6e320e4a6e..cf9f4718c2 100644 --- a/salt/salt/master.sls +++ b/salt/salt/master.sls @@ -1,16 +1,13 @@ -{% from 'salt/map.jinja' import SALTNOTHELD %} {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} include: - salt.minion -{% if SALTNOTHELD == 1 %} hold_salt_master_package: module.run: - pkg.hold: - name: salt-master -{% endif %} # prior to 2.4.30 this engine ran on the manager with salt-minion # this has changed to running with the salt-master in 2.4.30 diff --git a/salt/salt/minion.defaults.yaml b/salt/salt/minion.defaults.yaml index 2e4ebc93e5..be405b9e80 100644 --- a/salt/salt/minion.defaults.yaml +++ b/salt/salt/minion.defaults.yaml @@ -1,6 +1,6 @@ # version cannot be used elsewhere in this pillar as soup is grepping for it to determine if Salt needs to be patched salt: minion: - version: 3006.6 + version: 3006.9 check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default service_start_delay: 30 # in seconds. diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 8c6f7f0198..a5953e8e1a 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -2,13 +2,13 @@ {% from 'salt/map.jinja' import UPGRADECOMMAND with context %} {% from 'salt/map.jinja' import SALTVERSION %} {% from 'salt/map.jinja' import INSTALLEDSALTVERSION %} -{% from 'salt/map.jinja' import SALTNOTHELD %} {% from 'salt/map.jinja' import SALTPACKAGES %} {% from 'salt/map.jinja' import SYSTEMD_UNIT_FILE %} {% import_yaml 'salt/minion.defaults.yaml' as SALTMINION %} {% set service_start_delay = SALTMINION.salt.minion.service_start_delay %} include: + - salt.python_modules - salt - systemd.reload - repo.client @@ -19,15 +19,12 @@ include: {% if INSTALLEDSALTVERSION|string != SALTVERSION|string %} -{% if SALTNOTHELD | int == 0 %} unhold_salt_packages: - module.run: - - pkg.unhold: - - pkgs: + pkg.unheld: + - pkgs: {% for package in SALTPACKAGES %} - - {{ package }} + - {{ package }} {% endfor %} -{% endif %} install_salt_minion: cmd.run: @@ -41,15 +38,12 @@ install_salt_minion: {% if INSTALLEDSALTVERSION|string == SALTVERSION|string %} -{% if SALTNOTHELD | int == 1 %} hold_salt_packages: - module.run: - - pkg.hold: - - pkgs: + pkg.held: + - pkgs: {% for package in SALTPACKAGES %} - - {{ package }} + - {{ package }}: {{SALTVERSION}}-0.* {% endfor %} -{% endif %} remove_error_log_level_logfile: file.line: diff --git a/salt/salt/module_packages/docker/certifi-2024.7.4-py3-none-any.whl b/salt/salt/module_packages/docker/certifi-2024.7.4-py3-none-any.whl new file mode 100644 index 0000000000..9e412a2d94 Binary files /dev/null and b/salt/salt/module_packages/docker/certifi-2024.7.4-py3-none-any.whl differ diff --git a/salt/salt/module_packages/docker/charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/salt/salt/module_packages/docker/charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 0000000000..19a91a2781 Binary files /dev/null and b/salt/salt/module_packages/docker/charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/salt/salt/module_packages/docker/docker-7.1.0-py3-none-any.whl b/salt/salt/module_packages/docker/docker-7.1.0-py3-none-any.whl new file mode 100644 index 0000000000..c324efbd08 Binary files /dev/null and b/salt/salt/module_packages/docker/docker-7.1.0-py3-none-any.whl differ diff --git a/salt/salt/module_packages/docker/idna-3.7-py3-none-any.whl b/salt/salt/module_packages/docker/idna-3.7-py3-none-any.whl new file mode 100644 index 0000000000..fa4c95b123 Binary files /dev/null and b/salt/salt/module_packages/docker/idna-3.7-py3-none-any.whl differ diff --git a/salt/salt/module_packages/docker/requests-2.32.3-py3-none-any.whl b/salt/salt/module_packages/docker/requests-2.32.3-py3-none-any.whl new file mode 100644 index 0000000000..23662ce7ee Binary files /dev/null and b/salt/salt/module_packages/docker/requests-2.32.3-py3-none-any.whl differ diff --git a/salt/salt/module_packages/docker/urllib3-2.2.2-py3-none-any.whl b/salt/salt/module_packages/docker/urllib3-2.2.2-py3-none-any.whl new file mode 100644 index 0000000000..61a56086a5 Binary files /dev/null and b/salt/salt/module_packages/docker/urllib3-2.2.2-py3-none-any.whl differ diff --git a/salt/salt/python_modules.sls b/salt/salt/python_modules.sls new file mode 100644 index 0000000000..d6c05a892a --- /dev/null +++ b/salt/salt/python_modules.sls @@ -0,0 +1,21 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +docker_module_package: + file.recurse: + - name: /opt/so/conf/salt/module_packages/docker + - source: salt://salt/module_packages/docker + - clean: True + - makedirs: True + +# fail hard on this state so that soup would be cancelled on a manager (eventhough salt would have already updated) +# on a non manager, failing hard here will prevent the minion from upgrading +# we want to fail hard here to prevent the minion from upgrading and potetially being able to manager docker containers from a dep mismatch +docker_python_module_install: + cmd.run: + - name: /opt/saltstack/salt/bin/python3.10 -m pip install docker --no-index --find-links=/opt/so/conf/salt/module_packages/docker/ --upgrade + - onchanges: + - file: docker_module_package + - failhard: True diff --git a/salt/soc/config.sls b/salt/soc/config.sls index 8d1f0f694d..7607da5ff5 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -90,7 +90,7 @@ filedetectionsbackup: crondetectionsruntime: cron.present: - - name: /usr/sbin/so-detections-runtime-status cron + - name: /usr/sbin/so-detections-runtime-status cron - identifier: detections-runtime-status - user: root - minute: '*/10' @@ -190,6 +190,14 @@ socsigmarepo: - group: 939 - mode: 775 +socsensoronirepos: + file.directory: + - name: /opt/so/conf/soc/ai_summary_repos + - user: 939 + - group: 939 + - mode: 775 + - makedirs: True + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index f33783507d..f191fd0543 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1304,6 +1304,7 @@ soc: maxPacketCount: 5000 htmlDir: html importUploadDir: /nsm/soc/uploads + forceUserOtp: false modules: cases: soc filedatastore: @@ -1311,6 +1312,10 @@ soc: kratos: hostUrl: elastalertengine: + aiRepoUrl: https://github.com/Security-Onion-Solutions/securityonion-resources + aiRepoBranch: generated-summaries-stable + aiRepoPath: /opt/sensoroni/ai_summary_repos + showAiSummaries: true autoUpdateEnabled: true autoEnabledSigmaRules: default: @@ -1390,6 +1395,10 @@ soc: userFiles: - rbac/users_roles strelkaengine: + aiRepoUrl: https://github.com/Security-Onion-Solutions/securityonion-resources + aiRepoBranch: generated-summaries-stable + aiRepoPath: /opt/sensoroni/ai_summary_repos + showAiSummaries: true autoEnabledYaraRules: - securityonion-yara autoUpdateEnabled: true @@ -1411,6 +1420,10 @@ soc: stateFilePath: /opt/sensoroni/fingerprints/strelkaengine.state integrityCheckFrequencySeconds: 1200 suricataengine: + aiRepoUrl: https://github.com/Security-Onion-Solutions/securityonion-resources + aiRepoBranch: generated-summaries-stable + aiRepoPath: /opt/sensoroni/ai_summary_repos + showAiSummaries: true autoUpdateEnabled: true communityRulesImportFrequencySeconds: 86400 communityRulesImportErrorSeconds: 300 diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 9b50b449b8..99499115cc 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -33,6 +33,7 @@ so-soc: - /nsm/soc/uploads:/nsm/soc/uploads:rw - /opt/so/log/soc/:/opt/sensoroni/logs/:rw - /opt/so/conf/soc/soc.json:/opt/sensoroni/sensoroni.json:ro + - /opt/so/conf/soc/ai_summary_repos:/opt/sensoroni/ai_summary_repos:rw {% if SOCMERGED.telemetryEnabled and not GLOBALS.airgap %} - /opt/so/conf/soc/analytics.js:/opt/sensoroni/html/js/analytics.js:ro {% endif %} diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index b390c32c29..308044d504 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -81,15 +81,133 @@ soc: description: Maximum number of packets to show in the PCAP viewer. Larger values can cause more resource utilization on both the SOC server and the browser. global: True advanced: True + forceUserOtp: + title: Require TOTP + description: Require all users to enable Time-based One Time Passwords (MFA) upon login to SOC. + global: True modules: elastalertengine: + aiRepoUrl: + description: URL to the AI repository. This is used to pull in AI models for use in ElastAlert rules. + global: True + advanced: True + aiRepoBranch: + description: The branch to pull from the AI repository. Leaving this blank will pull the default branch. + global: True + advanced: True + aiRepoPath: + description: Path to the AI repository. This is used to pull in AI models for use in ElastAlert rules. + global: True + advanced: True + showAiSummaries: + description: Show AI summaries for ElastAlert rules. + global: True additionalAlerters: - title: Additional Alerters - description: Specify additional alerters to enable for all Sigma rules, one alerter name per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. Note that the configuration parameters for these alerters must be provided in the ElastAlert configuration section. Filter for 'Alerter' to find this related setting. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. + title: "Notifications: Sev 0/Default Alerters" + description: "Specify default alerters to enable for outbound notifications. These alerters will be used unless overridden by higher severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." global: True - helpLink: sigma.html + helpLink: notifications.html + forcedType: "[]string" + multiline: True + additionalSev0AlertersParams: + title: "Notifications: Sev 0/Default Parameters" + description: Optional configuration parameters for default alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. + global: True + multiline: True + syntax: yaml + helpLink: notifications.html + forcedType: string + additionalSev1Alerters: + title: "Notifications: Sev 1/Informational Alerters" + description: "Specify specific alerters to use when alerting at the info severity level or higher. These alerters will be used unless overridden by higher severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." + global: True + helpLink: notifications.html + forcedType: "[]string" + multiline: True + additionalSev1AlertersParams: + title: "Notifications: Sev 1/Informational Parameters" + description: Optional configuration parameters for informational severity alerters. Info level is less severe than 'Low Severity'. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. + global: True + multiline: True + syntax: yaml + helpLink: notifications.html + forcedType: string + additionalSev2Alerters: + title: "Notifications: Sev 2/Low Alerters" + description: "Specify specific alerters to use when alerting at the low severity level or higher. These alerters will be used unless overridden by higher severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." + global: True + helpLink: notifications.html + forcedType: "[]string" + multiline: True + additionalSev2AlertersParams: + title: "Notifications: Sev 2/Low Parameters" + description: Optional configuration parameters for low severity alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. + global: True + multiline: True + syntax: yaml + helpLink: notifications.html + forcedType: string + additionalSev3Alerters: + title: "Notifications: Sev 3/Medium Alerters" + description: "Specify specific alerters to use when alerting at the medium severity level or higher. These alerters will be used unless overridden by higher severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." + global: True + helpLink: notifications.html + forcedType: "[]string" + multiline: True + additionalSev3AlertersParams: + title: "Notifications: Sev 3/Medium Parameters" + description: Optional configuration parameters for medium severity alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. + global: True + multiline: True + syntax: yaml + helpLink: notifications.html + forcedType: string + additionalSev4Alerters: + title: "Notifications: Sev 4/High Alerters" + description: "Specify specific alerters to use when alerting at the high severity level or critical severity level. These alerters will be used unless overridden by critical severity alerter settings. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." + global: True + helpLink: notifications.html forcedType: "[]string" multiline: True + additionalSev4AlertersParams: + title: "Notifications: Sev 4/High Parameters" + description: Optional configuration parameters for high severity alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. + global: True + multiline: True + syntax: yaml + helpLink: notifications.html + forcedType: string + additionalSev5Alerters: + title: "Notifications: Sev 5/Critical Alerters" + description: "Specify specific alerters to use when alerting at the critical severity level. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." + global: True + helpLink: notifications.html + forcedType: "[]string" + multiline: True + additionalSev5AlertersParams: + title: "Notifications: Sev 5/Critical Parameters" + description: Optional configuration parameters for critical severity alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key. + global: True + multiline: True + syntax: yaml + helpLink: notifications.html + forcedType: string + additionalUserDefinedNotifications: + customAlerters: + description: "Specify custom notification alerters to use when the Sigma rule contains the following tag: so.alerters.customAlerters. This setting can be duplicated to create new custom alerter configurations. Specify one alerter name (Ex: 'email') per line. Alerters refers to ElastAlert 2 alerters, as documented at https://elastalert2.readthedocs.io. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." + global: True + helpLink: notifications.html + forcedType: "[]string" + duplicates: True + multiline: True + customAlertersParams: + description: "Optional configuration parameters for custom notification alerters, used when the Sigma rule contains the following tag: so.params.customAlertersParams. This setting can be duplicated to create new custom alerter configurations. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available alerters and their required configuration parameters. A full update of the ElastAlert rule engine, via the Detections screen, is required in order to apply these changes. Requires a valid Security Onion license key." + global: True + multiline: True + syntax: yaml + helpLink: notifications.html + duplicates: True + forcedType: string autoEnabledSigmaRules: default: &autoEnabledSigmaRules description: 'Sigma rules to automatically enable on initial import. Format is $Ruleset+$Level - for example, for the core community ruleset and critical level rules: core+critical. These will be applied based on role if defined and default if not.' @@ -189,6 +307,21 @@ soc: advanced: True forcedType: int strelkaengine: + aiRepoUrl: + description: URL to the AI repository. This is used to pull in AI models for use in Strelka rules. + global: True + advanced: True + aiRepoBranch: + description: The branch to pull from the AI repository. Leaving this blank will pull the default branch. + global: True + advanced: True + aiRepoPath: + description: Path to the AI repository. This is used to pull in AI models for use in Strelka rules. + global: True + advanced: True + showAiSummaries: + description: Show AI summaries for Strelka rules. + global: True autoEnabledYaraRules: description: 'YARA rules to automatically enable on initial import. Format is $Ruleset - for example, for the default shipped ruleset: securityonion-yara' global: True @@ -212,6 +345,21 @@ soc: helpLink: yara.html airgap: *serulesRepos suricataengine: + aiRepoUrl: + description: URL to the AI repository. This is used to pull in AI models for use in Suricata rules. + global: True + advanced: True + aiRepoBranch: + description: The branch to pull from the AI repository. Leaving this blank will pull the default branch. + global: True + advanced: True + aiRepoPath: + description: Path to the AI repository. This is used to pull in AI models for use in Suricata rules. + global: True + advanced: True + showAiSummaries: + description: Show AI summaries for Suricata rules. + global: True communityRulesImportFrequencySeconds: description: 'How often to check for new Suricata rules (in seconds).' global: True diff --git a/salt/suricata/map.jinja b/salt/suricata/map.jinja index a5012317a2..a2c7072e05 100644 --- a/salt/suricata/map.jinja +++ b/salt/suricata/map.jinja @@ -101,20 +101,20 @@ {# change address-groups vars from list to comma seperated string #} {% for k, v in SURICATAMERGED.config.vars['address-groups'].items() %} +{% if v is string %} +{% do SURICATAMERGED.config.vars['address-groups'].update({k: '[' ~ v ~ ']'}) %} {# if address-group value is a list #} -{% if v is iterable and (v is not string and v is not mapping and v | length > 1) %} +{% elif v is iterable and v is not mapping %} {% do SURICATAMERGED.config.vars['address-groups'].update({k: '[' ~ v | join(',') ~ ']'}) %} -{% else %} -{% do SURICATAMERGED.config.vars['address-groups'].update({k: v[0]}) %} {% endif %} {% endfor %} {# change port-groups vars from list to comma seperated string #} {% for k, v in SURICATAMERGED.config.vars['port-groups'].items() %} +{% if v is string %} +{% do SURICATAMERGED.config.vars['port-groups'].update({k: '[' ~ v ~ ']'}) %} {# if address-group value is a list #} -{% if v is iterable and (v is not string and v is not mapping and v | length > 1) %} +{% elif v is iterable and v is not mapping %} {% do SURICATAMERGED.config.vars['port-groups'].update({k: '[' ~ v | join(',') ~ ']'}) %} -{% else %} -{% do SURICATAMERGED.config.vars['port-groups'].update({k: v[0]}) %} {% endif %} {% endfor %} diff --git a/setup/files/salt_module_deps/docker/certifi-2022.12.7-py3-none-any.whl b/setup/files/salt_module_deps/docker/certifi-2022.12.7-py3-none-any.whl deleted file mode 100644 index a083056113..0000000000 Binary files a/setup/files/salt_module_deps/docker/certifi-2022.12.7-py3-none-any.whl and /dev/null differ diff --git a/setup/files/salt_module_deps/docker/chardet-4.0.0-py2.py3-none-any.whl b/setup/files/salt_module_deps/docker/chardet-4.0.0-py2.py3-none-any.whl deleted file mode 100644 index b83344e8e3..0000000000 Binary files a/setup/files/salt_module_deps/docker/chardet-4.0.0-py2.py3-none-any.whl and /dev/null differ diff --git a/setup/files/salt_module_deps/docker/charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/setup/files/salt_module_deps/docker/charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index 7b57bc7167..0000000000 Binary files a/setup/files/salt_module_deps/docker/charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/setup/files/salt_module_deps/docker/docker-5.0.2-py2.py3-none-any.whl b/setup/files/salt_module_deps/docker/docker-5.0.2-py2.py3-none-any.whl deleted file mode 100644 index f1ed95ee12..0000000000 Binary files a/setup/files/salt_module_deps/docker/docker-5.0.2-py2.py3-none-any.whl and /dev/null differ diff --git a/setup/files/salt_module_deps/docker/idna-2.10-py2.py3-none-any.whl b/setup/files/salt_module_deps/docker/idna-2.10-py2.py3-none-any.whl deleted file mode 100644 index 41225cb056..0000000000 Binary files a/setup/files/salt_module_deps/docker/idna-2.10-py2.py3-none-any.whl and /dev/null differ diff --git a/setup/files/salt_module_deps/docker/requests-2.25.1-py2.py3-none-any.whl b/setup/files/salt_module_deps/docker/requests-2.25.1-py2.py3-none-any.whl deleted file mode 100644 index 8d70e9716d..0000000000 Binary files a/setup/files/salt_module_deps/docker/requests-2.25.1-py2.py3-none-any.whl and /dev/null differ diff --git a/setup/files/salt_module_deps/docker/urllib3-1.26.15-py2.py3-none-any.whl b/setup/files/salt_module_deps/docker/urllib3-1.26.15-py2.py3-none-any.whl deleted file mode 100644 index ad723d5e10..0000000000 Binary files a/setup/files/salt_module_deps/docker/urllib3-1.26.15-py2.py3-none-any.whl and /dev/null differ diff --git a/setup/files/salt_module_deps/docker/websocket_client-1.5.1-py3-none-any.whl b/setup/files/salt_module_deps/docker/websocket_client-1.5.1-py3-none-any.whl deleted file mode 100644 index bd8152d2c4..0000000000 Binary files a/setup/files/salt_module_deps/docker/websocket_client-1.5.1-py3-none-any.whl and /dev/null differ diff --git a/setup/files/salt_module_deps/pymysql/PyMySQL-1.0.3-py3-none-any.whl b/setup/files/salt_module_deps/pymysql/PyMySQL-1.0.3-py3-none-any.whl deleted file mode 100644 index 0bbd29da02..0000000000 Binary files a/setup/files/salt_module_deps/pymysql/PyMySQL-1.0.3-py3-none-any.whl and /dev/null differ diff --git a/setup/so-functions b/setup/so-functions index 4fe5ddebce..23b16156d5 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1814,7 +1814,7 @@ repo_sync_local() { if [[ ! $is_airgap ]]; then curl --retry 5 --retry-delay 60 -A "netinstall/$SOVERSION/$OS/$(uname -r)/1" https://sigs.securityonion.net/checkup --output /tmp/install - logCmd "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/" + retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/" >> "$setup_log" 2>&1 || fail_setup # After the download is complete run createrepo create_repo fi @@ -1931,7 +1931,7 @@ saltify() { } salt_install_module_deps() { - logCmd "salt-pip install docker --no-index --only-binary=:all: --find-links files/salt_module_deps/docker/" + logCmd "salt-call state.apply salt.python_modules --local --file-root=../salt/" } salt_patch_x509_v2() { diff --git a/sigs/securityonion-2.4.100-20240829.iso.sig b/sigs/securityonion-2.4.100-20240829.iso.sig new file mode 100644 index 0000000000..39db1a63d8 Binary files /dev/null and b/sigs/securityonion-2.4.100-20240829.iso.sig differ