Merge pull request #862 from Security-Onion-Solutions/dev

1.4.0

README.md

@@ -1,3 +1,37 @@
+## Hybrid Hunter Beta 1.4.0 - Beta 3
+
+- Complete overhaul of the way we handle custom and default settings and data. You will now see a default and local directory under the saltstack directory. All customizations are stored in local.
+- The way firewall rules are handled has been completely revamped. This will allow the user to customize firewall rules much easier. 
+- Users can now change their own password in SOC.
+- Hunt now allows users to enable auto-hunt. This is a toggle which, when enabled, automatically submits a new hunt when filtering, grouping, etc.
+- Title bar now reflects current Hunt query. This will assist users in locating a previous query from their browser history.
+- Zeek 3.0.7
+- Elastic 7.7.1
+- Suricata can now be used for meta data generation.
+- Suricata eve.json has been moved to `/nsm` to align with storage of other data.
+- Suricata will now properly rotate its logs.
+- Grafana dashboards now work properly in standalone mode.
+- Kibana Dashboard updates including osquery, community_id.  
+- New Elasticsearch Ingest processor to generate community_id from any log that includes the required fields.  
+- Community_id generated for additional logs: Zeek HTTP/SMTP/ , Sysmon shipped with Osquery or Winlogbeat.  
+- Major streamlining of Fleet setup & configuration - no need to run a secondary setup script anymore. 
+- Fleet Standalone node now includes the ability to set a FQDN to point osquery endpoints to. 
+- Distributed installs now support ingesting Windows Eventlogs via Winlogbeat - includes full parsing support for Sysmon.
+- SOC Downloads section now includes a link to the supported version of Winlogbeat.     
+- Basic syslog ingestion capability now included.
+- Elasticsearch index name transition fixes for various components.
+- Updated URLs for pivot fields in Kibana.
+- Instances of `hive` renamed to `thehive`. 
+
+### Known Issues:
+
+- The Hunt feature is currently considered "Preview" and although very useful in its current state, not everything works. We wanted to get this out as soon as possible to get the feedback from you! Let us know what you want to see! Let us know what you think we should call it!
+- You cannot pivot to PCAP from Suricata alerts in Kibana or Hunt.
+- Navigator is currently not working when using hostname to access SOC. IP mode works correctly.
+- Due to the move to ECS, the current Playbook plays may not alert correctly at this time.
+- The osquery MacOS package does not install correctly.
+
+
 ## Hybrid Hunter Beta 1.3.0 - Beta 2
 
 ### Changes:

VERSION

@@ -1 +1 @@
-1.3.0
+1.4.0

files/firewall/assigned_hostgroups.local.map.yaml

@@ -0,0 +1,20 @@
+{% import_yaml 'firewall/portgroups.yaml' as default_portgroups %}
+{% set default_portgroups = default_portgroups.firewall.aliases.ports %}
+{% import_yaml 'firewall/portgroups.local.yaml' as local_portgroups %}
+{% if local_portgroups.firewall.aliases.ports %}
+  {% set local_portgroups = local_portgroups.firewall.aliases.ports %}
+{% else %}
+  {% set local_portgroups = {} %}
+{% endif %}
+{% set portgroups = salt['defaults.merge'](default_portgroups, local_portgroups, in_place=False) %}
+
+role:
+  eval:
+  fleet:
+  heavynode:
+  helixsensor:
+  master:
+  mastersearch:
+  standalone:
+  searchnode:
+  sensor:

files/firewall/hostgroups.local.yaml

@@ -0,0 +1,62 @@
+firewall:
+  hostgroups:
+    analyst:
+      ips:
+        delete:
+        insert:
+    beats_endpoint:
+      ips:
+        delete:
+        insert:
+    beats_endpoint_ssl:
+      ips:
+        delete:
+        insert:
+    fleet:
+      ips:
+        delete:
+        insert:
+    heavy_node:
+      ips:
+        delete:
+        insert:
+    master:
+      ips:
+        delete:
+        insert:
+    minion:
+      ips:
+        delete:
+        insert:
+    node:
+      ips:
+        delete:
+        insert:
+    osquery_endpoint:
+      ips:
+        delete:
+        insert:
+    search_node:
+      ips:
+        delete:
+        insert:
+    sensor:
+      ips:
+        delete:
+        insert:
+    syslog:
+      ips:
+        delete:
+        insert:
+    wazuh_agent:
+      ips:
+        delete:
+        insert:
+    wazuh_api:
+      ips:
+        delete:
+        insert:
+    wazuh_authd:
+      ips:
+        delete:
+        insert:

files/firewall/portgroups.local.yaml

@@ -0,0 +1,3 @@
+firewall:
+  aliases:
+    ports:

files/master

@@ -37,7 +37,9 @@ log_file: /opt/so/log/salt/master
 #
 file_roots:
   base:
-    - /opt/so/saltstack/salt
+    - /opt/so/saltstack/local/salt
+    - /opt/so/saltstack/default/salt
+
 
 # The master_roots setting configures a master-only copy of the file_roots dictionary,
 # used by the state compiler.
@@ -53,7 +55,8 @@ file_roots:
 
 pillar_roots:
   base:
-    - /opt/so/saltstack/pillar
+    - /opt/so/saltstack/local/pillar
+    - /opt/so/saltstack/default/pillar
 
 peer:
   .*:

pillar/data/addtotab.sh

@@ -1,7 +1,8 @@
 #!/usr/bin/env bash
 
 # This script adds sensors/nodes/etc to the nodes tab
-
+default_salt_dir=/opt/so/saltstack/default
+local_salt_dir=/opt/so/saltstack/local
 TYPE=$1
 NAME=$2
 IPADDRESS=$3
@@ -15,7 +16,7 @@ MONINT=$9
 #HOTNAME=$11
 
 echo "Seeing if this host is already in here. If so delete it"
-if grep -q $NAME "/opt/so/saltstack/pillar/data/$TYPE.sls"; then
+if grep -q $NAME "$local_salt_dir/pillar/data/$TYPE.sls"; then
   echo "Node Already Present - Let's re-add it"
   awk -v blah="  $NAME:" 'BEGIN{ print_flag=1 }
 {
@@ -31,27 +32,29 @@ if grep -q $NAME "/opt/so/saltstack/pillar/data/$TYPE.sls"; then
     if ( print_flag == 1 )
         print $0
 
-} ' /opt/so/saltstack/pillar/data/$TYPE.sls > /opt/so/saltstack/pillar/data/tmp.$TYPE.sls
-mv /opt/so/saltstack/pillar/data/tmp.$TYPE.sls /opt/so/saltstack/pillar/data/$TYPE.sls
+} ' $local_salt_dir/pillar/data/$TYPE.sls > $local_salt_dir/pillar/data/tmp.$TYPE.sls
+mv $local_salt_dir/pillar/data/tmp.$TYPE.sls $local_salt_dir/pillar/data/$TYPE.sls
 echo "Deleted $NAME from the tab. Now adding it in again with updated info"
 fi
-echo "  $NAME:" >> /opt/so/saltstack/pillar/data/$TYPE.sls
-echo "    ip: $IPADDRESS" >> /opt/so/saltstack/pillar/data/$TYPE.sls
-echo "    manint: $MANINT" >> /opt/so/saltstack/pillar/data/$TYPE.sls
-echo "    totalcpus: $CPUS" >> /opt/so/saltstack/pillar/data/$TYPE.sls
-echo "    guid: $GUID" >> /opt/so/saltstack/pillar/data/$TYPE.sls
-echo "    rootfs: $ROOTFS" >> /opt/so/saltstack/pillar/data/$TYPE.sls
-echo "    nsmfs: $NSM" >> /opt/so/saltstack/pillar/data/$TYPE.sls
+echo "  $NAME:" >> $local_salt_dir/pillar/data/$TYPE.sls
+echo "    ip: $IPADDRESS" >> $local_salt_dir/pillar/data/$TYPE.sls
+echo "    manint: $MANINT" >> $local_salt_dir/pillar/data/$TYPE.sls
+echo "    totalcpus: $CPUS" >> $local_salt_dir/pillar/data/$TYPE.sls
+echo "    guid: $GUID" >> $local_salt_dir/pillar/data/$TYPE.sls
+echo "    rootfs: $ROOTFS" >> $local_salt_dir/pillar/data/$TYPE.sls
+echo "    nsmfs: $NSM" >> $local_salt_dir/pillar/data/$TYPE.sls
 if [ $TYPE == 'sensorstab' ]; then
-  echo "    monint: $MONINT" >> /opt/so/saltstack/pillar/data/$TYPE.sls
-  salt-call state.apply common queue=True
+  echo "    monint: $MONINT" >> $local_salt_dir/pillar/data/$TYPE.sls
+  salt-call state.apply grafana queue=True
 fi
-if [ $TYPE == 'evaltab' ]; then
-  echo "    monint: $MONINT" >> /opt/so/saltstack/pillar/data/$TYPE.sls
-  salt-call state.apply common queue=True
-  salt-call state.apply utility queue=True
+if [ $TYPE == 'evaltab' ] || [ $TYPE == 'standalonetab' ]; then
+  echo "    monint: $MONINT" >> $local_salt_dir/pillar/data/$TYPE.sls
+  if [ ! $10 ]; then
+    salt-call state.apply grafana queue=True
+    salt-call state.apply utility queue=True
+  fi
 fi
 #if [ $TYPE == 'nodestab' ]; then
-#  echo "    nodetype: $NODETYPE" >> /opt/so/saltstack/pillar/data/$TYPE.sls
-#  echo "    hotname: $HOTNAME" >> /opt/so/saltstack/pillar/data/$TYPE.sls
+#  echo "    nodetype: $NODETYPE" >> $local_salt_dir/pillar/data/$TYPE.sls
+#  echo "    hotname: $HOTNAME" >> $local_salt_dir/pillar/data/$TYPE.sls
 #fi

pillar/firewall/addfirewall.sh

@@ -1,13 +1,13 @@
 #!/usr/bin/env bash
 
 # This script adds ip addresses to specific rule sets defined by the user
-
+local_salt_dir=/opt/so/saltstack/local
 POLICY=$1
 IPADDRESS=$2
 
-if grep -q $2 "/opt/so/saltstack/pillar/firewall/$1.sls"; then
+if grep -q $2 "$local_salt_dir/pillar/firewall/$1.sls"; then
   echo "Firewall Rule Already There"
 else
-  echo "  - $2" >> /opt/so/saltstack/pillar/firewall/$1.sls
+  echo "  - $2" >> $local_salt_dir/pillar/firewall/$1.sls
   salt-call state.apply firewall queue=True
-fi
+fi

pillar/firewall/ports.sls

@@ -0,0 +1,62 @@
+firewall:
+  analyst:
+    ports:
+      tcp:
+        - 80
+        - 443
+      udp:
+  beats_endpoint:
+    ports:
+      tcp:
+        - 5044
+  forward_nodes:
+    ports:
+      tcp:
+        - 443
+        - 5044
+        - 5644
+        - 9822
+      udp:
+  master:
+    ports:
+      tcp:
+        - 1514
+        - 3200
+        - 3306
+        - 4200
+        - 5601
+        - 6379
+        - 8086
+        - 8090
+        - 9001
+        - 9200
+        - 9300
+        - 9400  
+        - 9500
+      udp:
+        - 1514
+  minions:
+    ports:
+      tcp:
+        - 3142
+        - 4505
+        - 4506
+        - 5000
+        - 8080
+        - 8086
+        - 55000      
+  osquery_endpoint:
+    ports:
+      tcp:
+        - 8090
+  search_nodes:
+    ports:
+      tcp:
+        - 6379
+        - 9300
+  wazuh_endpoint:
+    ports:
+      tcp:
+        - 1514
+      udp: 
+        -1514

pillar/logstash/master.sls

@@ -2,5 +2,6 @@ logstash:
   pipelines:
     master:
       config:
+        - so/0009_input_beats.conf      
         - so/0010_input_hhbeats.conf
         - so/9999_output_redis.conf.jinja

pillar/logstash/search.sls

@@ -5,12 +5,12 @@ logstash:
         - so/0900_input_redis.conf.jinja
         - so/9000_output_zeek.conf.jinja
         - so/9002_output_import.conf.jinja
+        - so/9034_output_syslog.conf.jinja
         - so/9100_output_osquery.conf.jinja
         - so/9400_output_suricata.conf.jinja
         - so/9500_output_beats.conf.jinja
         - so/9600_output_ossec.conf.jinja
         - so/9700_output_strelka.conf.jinja
   templates:
-    - so/so-beats-template.json
     - so/so-common-template.json
     - so/so-zeek-template.json

pillar/top.sls

@@ -14,15 +14,13 @@ base:
 
   '*_sensor':
     - static
-    - firewall.*
     - brologs
     - healthcheck.sensor
     - minions.{{ grains.id }}
 
   '*_master or *_mastersearch':
     - match: compound
     - static
-    - firewall.*
     - data.*
     - secrets
     - minions.{{ grains.id }}
@@ -33,7 +31,6 @@ base:
 
   '*_eval':
     - static
-    - firewall.*
     - data.*
     - brologs
     - secrets
@@ -44,7 +41,6 @@ base:
     - logstash
     - logstash.master
     - logstash.search
-    - firewall.*
     - data.*
     - brologs
     - secrets
@@ -54,18 +50,15 @@ base:
 
   '*_node':
     - static
-    - firewall.*
     - minions.{{ grains.id }}
 
   '*_heavynode':
     - static
-    - firewall.*
     - brologs
     - minions.{{ grains.id }}
 
   '*_helix':
     - static
-    - firewall.*
     - fireeye
     - brologs
     - logstash
@@ -74,14 +67,12 @@ base:
 
   '*_fleet':
     - static
-    - firewall.*
     - data.*
     - secrets
     - minions.{{ grains.id }}
 
   '*_searchnode':
     - static
-    - firewall.*
     - logstash
     - logstash.search
     - minions.{{ grains.id }}