Merge branch '2.4/dev' into kilo

pillar/top.sls

@@ -10,6 +10,7 @@ base:
     - sensoroni.adv_sensoroni
     - telegraf.soc_telegraf
     - telegraf.adv_telegraf
+    - influxdb.token
     - node_data.ips
 
   '* and not *_eval and not *_import':
@@ -27,6 +28,8 @@ base:
     - logstash.soc_logstash
     - logstash.adv_logstash
     - elasticsearch.index_templates
+    - elasticsearch.soc_elasticsearch
+    - elasticsearch.adv_elasticsearch
 
   '*_manager':
     - logstash

salt/firewall/assigned_hostgroups.map.yaml

@@ -411,11 +411,9 @@ role:
           elasticsearch_rest:
             portgroups:
               - {{ portgroups.elasticsearch_rest }}
-          {% if TRUE_CLUSTER %}
           searchnodes:
             portgroups:
               - {{ portgroups.elasticsearch_node }}
-          {% endif %}
           self:
             portgroups:
               - {{ portgroups.syslog}}
@@ -469,6 +467,8 @@ role:
           self:
             portgroups:
               - {{ portgroups.syslog}}
+              - {{ portgroups.elasticsearch_node }}
+              - {{ portgroups.elasticsearch_rest }}
           strelka_frontend:
             portgroups:
               - {{ portgroups.strelka_frontend }}

salt/influxdb/curl.config.jinja

@@ -1 +1 @@
-header = "Authorization: Token {{ salt['pillar.get']('secrets:influx_token') }}"
+header = "Authorization: Token {{ salt['pillar.get']('influxdb:token') }}"

salt/influxdb/init.sls

@@ -6,7 +6,7 @@
 
 {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-eval', 'so-import'] %}
 {% set PASSWORD = salt['pillar.get']('secrets:influx_pass') %}
-{% set TOKEN = salt['pillar.get']('secrets:influx_token') %}
+{% set TOKEN = salt['pillar.get']('influxdb:token') %}
 
 include:
   - salt.minion

salt/soc/defaults.map.jinja

@@ -1,7 +1,7 @@
 {% import_yaml 'soc/defaults.yaml' as SOCDEFAULTS %}
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% from 'docker/docker.map.jinja' import DOCKER -%}
-{% set INFLUXDB_TOKEN = salt['pillar.get']('secrets:influx_token') %}
+{% set INFLUXDB_TOKEN = salt['pillar.get']('influxdb:token') %}
 {% import_text 'influxdb/metrics_link.txt' as METRICS_LINK %}
 
 {% for module, application_url in GLOBALS.application_urls.items() %}

salt/soc/defaults.yaml

@@ -7,19 +7,19 @@ soc:
       icon: fa-crosshairs
       target:
       links:
-        - '/#/hunt?q="{value|escape}" | groupby event.module event.dataset'
+        - '/#/hunt?q="{value|escape}" | groupby event.module* event.dataset'
     - name: actionCorrelate
       description: actionCorrelateHelp
       icon: fab fa-searchengin
       target: ''
       links:
-        - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}" OR "{:network.community_id}") | groupby event.module event.dataset'
+        - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}" OR "{:network.community_id}") | groupby event.module* event.dataset'
-        - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}") | groupby event.module event.dataset'
+        - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}") | groupby event.module* event.dataset'
-        - '/#/hunt?q=("{:log.id.fuid}" OR "{:network.community_id}") | groupby event.module event.dataset'
+        - '/#/hunt?q=("{:log.id.fuid}" OR "{:network.community_id}") | groupby event.module* event.dataset'
-        - '/#/hunt?q=("{:log.id.uid}" OR "{:network.community_id}") | groupby event.module event.dataset'
+        - '/#/hunt?q=("{:log.id.uid}" OR "{:network.community_id}") | groupby event.module* event.dataset'
-        - '/#/hunt?q="{:log.id.fuid}" | groupby event.module event.dataset'
+        - '/#/hunt?q="{:log.id.fuid}" | groupby event.module* event.dataset'
-        - '/#/hunt?q="{:log.id.uid}" | groupby event.module event.dataset'
+        - '/#/hunt?q="{:log.id.uid}" | groupby event.module* event.dataset'
-        - '/#/hunt?q="{:network.community_id}" | groupby event.module event.dataset'
+        - '/#/hunt?q="{:network.community_id}" | groupby event.module* event.dataset'
     - name: actionPcap
       description: actionPcapHelp
       icon: fa-stream
@@ -560,13 +560,13 @@ soc:
       - destination.geo.country_iso_code
       - user.name
       - source.ip
-    ':sysmon:':
+    ':windows.sysmon_operational:':
       - soc_timestamp
-      - event.dataset
+      - event.action
       - process.executable
       - user.name
       - file.target
-      - dns.query.name
+      - dns.question.name
       - winlog.event_data.TargetObject
     '::network_connection':
       - soc_timestamp
@@ -1116,12 +1116,12 @@ soc:
             enabled: true
         queries:
           - name: Default Query
-            description: Show all events grouped by the origin host
+            description: Show all events grouped by the observer host
             query: '* | groupby observer.name'
             showSubtitle: true
           - name: Log Type
             description: Show all events grouped by module and dataset
-            query: '* | groupby event.module event.dataset'
+            query: '* | groupby event.module* event.dataset'
             showSubtitle: true
           - name: SOC Auth
             description: Users authenticated to SOC grouped by IP address and identity
@@ -1145,11 +1145,11 @@ soc:
             showSubtitle: true
           - name: Sysmon Events
             description: Show all Sysmon logs grouped by event type
-            query: 'event.module:sysmon | groupby event.dataset'
+            query: 'event.dataset: windows.sysmon_operational | groupby event.action'
             showSubtitle: true
           - name: Sysmon Usernames
             description: Show all Sysmon logs grouped by username
-            query: 'event.module:sysmon | groupby event.dataset, user.name.keyword'
+            query: 'event.dataset: windows.sysmon_operational | groupby event.action, user.name.keyword'
             showSubtitle: true
           - name: Strelka
             description: Show all Strelka logs grouped by file type
@@ -1380,7 +1380,7 @@ soc:
         queries:
           - name: Overview
             description: Overview of all events
-            query: '* | groupby -sankey event.dataset event.category* | groupby -pie event.category | groupby -bar event.module | groupby event.dataset | groupby event.module | groupby event.category | groupby observer.name | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name'
+            query: '* | groupby -sankey event.dataset event.category* | groupby -pie event.category | groupby -bar event.module* | groupby event.dataset | groupby event.module* | groupby event.category | groupby observer.name | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name'
           - name: SOC Auth
             description: SOC (Security Onion Console) authentication logs
             query: 'event.module:kratos AND event.dataset:audit AND msg:authenticated | groupby -sankey http_request.headers.x-real-ip identity_id | groupby http_request.headers.x-real-ip | groupby identity_id | groupby http_request.headers.user-agent'
@@ -1389,28 +1389,31 @@ soc:
             query: '_index: "*:elastalert*" | groupby rule_name | groupby alert_info.type'
           - name: Alerts
             description: Overview of all alerts
-            query: 'event.dataset:alert | groupby event.module | groupby rule.name | groupby event.severity | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name'
+            query: 'event.dataset:alert | groupby event.module* | groupby rule.name | groupby event.severity | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name'
           - name: NIDS Alerts
             description: NIDS (Network Intrusion Detection System) alerts
             query: 'event.category:network AND event.dataset:alert | groupby rule.category | groupby -sankey source.ip destination.ip | groupby rule.name | groupby rule.uuid | groupby rule.gid | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name'
           - name: Sysmon Overview
             description: Overview of all Sysmon data types
-            query: 'event.module:sysmon | groupby -sankey event.dataset winlog.computer_name | groupby -sankey winlog.computer_name user.name | groupby winlog.computer_name | groupby event.dataset | groupby user.name | groupby dns.query.name | groupby process.executable | groupby winlog.event_data.TargetObject | groupby file.target | groupby source.ip | groupby destination.ip | groupby destination.port'
+            query: 'event.dataset:windows.sysmon_operational | groupby -sankey event.action host.name | groupby -sankey host.name user.name | groupby host.name | groupby event.category event.action  | groupby user.name | groupby dns.question.name | groupby process.executable | groupby winlog.event_data.TargetObject | groupby file.name | groupby source.ip | groupby destination.ip | groupby destination.port'
-          - name: Sysmon Registry
+          - name: Host Overview
-            description: Registry changes captured by Sysmon
+            description: Overview of all host data types
-            query: '(event.dataset:registry_create_delete OR event.dataset:registry_value_set OR event.dataset:registry_key_value_rename) | groupby -sankey event.dataset winlog.computer_name | groupby winlog.computer_name | groupby event.dataset | groupby process.executable | groupby winlog.event_data.TargetObject | groupby process.executable winlog.event_data.TargetObject'
+            query: '((event.category:registry OR event.category:host OR event.category:process OR event.category:driver OR event.category:configuration) OR (event.category:file  AND _exists_:process.executable) OR (event.category:network AND _exists_:host.name)) | groupby event.dataset* event.category* event.action* | groupby event.type | groupby host.name | groupby user.name | groupby file.name | groupby process.executable'
-          - name: Sysmon DNS
+          - name: Host Registry Changes
-            description: DNS queries captured by Sysmon
+            description: Windows Registry changes
-            query: 'event.dataset:dns_query | groupby -sankey winlog.computer_name dns.query.name | groupby winlog.computer_name | groupby process.executable | groupby dns.query.name | groupby dns.answers.name'
+            query: 'event.category: registry | groupby -sankey event.action host.name | groupby event.dataset event.action | groupby host.name | groupby process.executable | groupby registry.path | groupby process.executable registry.path'
-          - name: Sysmon Process
+          - name: Host DNS & Process Mappings 
-            description: Process activity captured by Sysmon
+            description: DNS queries mapped to originating processes
-            query: '(event.dataset:process_creation OR event.dataset:process_terminated OR event.dataset:process_access) | groupby -sankey winlog.computer_name user.name | groupby winlog.computer_name | groupby user.name | groupby event.dataset | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable'
+            query: 'event.category: network AND _exists_:process.executable  AND (_exists_:dns.question.name OR _exists_:dns.answers.data)  | groupby -sankey host.name dns.question.name | groupby event.dataset event.type | groupby host.name | groupby process.executable | groupby dns.question.name | groupby dns.answers.data'
-          - name: Sysmon File
+          - name: Host Process Activity
-            description: File activity captured by Sysmon
+            description: Process activity captured on an endpoint
-            query: 'event.module:sysmon AND event.dataset:file_* | groupby -sankey winlog.computer_name process.executable | groupby winlog.computer_name | groupby event.dataset | groupby file.target | groupby process.executable'
+            query: 'event.category:process | groupby -sankey host.name user.name*  | groupby event.dataset event.action | groupby host.name | groupby user.name | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable'
-          - name: Sysmon Network
+          - name: Host File Activity
-            description: Network activity captured by Sysmon
+            description: File activity captured on an endpoint
-            query: 'event.dataset:network_connection | groupby -sankey winlog.computer_name destination.ip destination.port | groupby winlog.computer_name | groupby user.name | groupby process.executable | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name'
+            query: 'event.category: file AND _exists_:process.executable | groupby -sankey host.name process.executable | groupby host.name | groupby event.dataset event.action event.type | groupby file.name | groupby process.executable'
+          - name: Host Network & Process Mappings
+            description: Network activity mapped to originating processes
+            query: 'event.category: network AND _exists_:process.executable | groupby -sankey event.action host.name | groupby -sankey host.name user.name | groupby event.dataset* event.type* event.action* | groupby host.name | groupby user.name | groupby dns.question.name | groupby process.executable | groupby winlog.event_data.TargetObject | groupby process.name | groupby source.ip | groupby destination.ip | groupby destination.port'
           - name: Strelka
             description: Strelka file analysis
             query: 'event.module:strelka | groupby file.mime_type | groupby -sankey file.mime_type file.source | groupby file.source | groupby file.name'
@@ -1614,7 +1617,7 @@ soc:
               - acknowledged
         queries:
           - name: 'Group By Name, Module'
-            query: '* | groupby rule.name event.module event.severity_label'
+            query: '* | groupby rule.name event.module* event.severity_label'
           - name: 'Group By Sensor, Source IP/Port, Destination IP/Port, Name'
             query: '* | groupby observer.name source.ip source.port destination.ip destination.port rule.name network.community_id event.severity_label'
           - name: 'Group By Source IP, Name'

salt/telegraf/etc/telegraf.conf

@@ -2,7 +2,7 @@
 {%- set INFLUXDBHOST = GLOBALS.influxdb_host %}
 {%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
-{%- set TOKEN = salt['pillar.get']('secrets:influx_token', '') %}
+{%- set TOKEN = salt['pillar.get']('influxdb:token', '') %}
 {%- set NODEIP = GLOBALS.node_ip %}
 {%- set UNIQUEID = salt['pillar.get']('sensor:uniqueid', '') %}
 {%- set ZEEK_ENABLED = salt['pillar.get']('zeek:enabled', True) %}

salt/top.sls

@@ -147,6 +147,7 @@ base:
     - schedule
     - soctopus
     - playbook
+    - elastic-fleet
     - docker_clean
 
   '*_standalone and G@saltversion:{{saltversion}}':
@@ -197,8 +198,8 @@ base:
     - schedule
     - soctopus
     - playbook
-    - docker_clean
     - elastic-fleet
+    - docker_clean
 
   '*_searchnode and G@saltversion:{{saltversion}}':
     - match: compound
@@ -256,6 +257,7 @@ base:
     - schedule
     - soctopus
     - playbook
+    - elastic-fleet
     - docker_clean
 
   '*_heavynode and G@saltversion:{{saltversion}}':
@@ -315,8 +317,8 @@ base:
     - suricata
     - zeek
     - schedule
-    - docker_clean
     - elastic-fleet
+    - docker_clean
 
   '*_receiver and G@saltversion:{{saltversion}}':
     - match: compound

salt/vars/heavynode.map.jinja

@@ -0,0 +1,17 @@
+{% from 'vars/elasticsearch.map.jinja' import ELASTICSEARCH_GLOBALS %}
+{% from 'vars/logstash.map.jinja' import LOGSTASH_GLOBALS %}
+{% from 'vars/sensor.map.jinja' import SENSOR_GLOBALS %}
+
+{% set ROLE_GLOBALS = {} %}
+
+{% set HEAVYNODE_GLOBALS =
+   [
+     SENSOR_GLOBALS,
+     ELASTICSEARCH_GLOBALS,
+     LOGSTASH_GLOBALS
+   ]
+%}
+
+{% for sg in HEAVYNODE_GLOBALS %}
+{%   do salt['defaults.merge'](ROLE_GLOBALS, sg, merge_lists=False, in_place=True) %}
+{% endfor %}

setup/so-functions

@@ -897,6 +897,7 @@ create_manager_pillars() {
 	kratos_pillar
 	soc_pillar
 	idh_pillar
+	influxdb_pillar
 
 }
 
@@ -1536,6 +1537,9 @@ influxdb_pillar() {
 	title "Create the influxdb pillar file"
 	touch $adv_influxdb_pillar_file
 	touch $influxdb_pillar_file
+	printf '%s\n'\
+		"influxdb:"\
+		"  token: $INFLUXTOKEN" > $local_salt_dir/pillar/influxdb/token.sls
 }
 
 make_some_dirs() {
@@ -1711,7 +1715,7 @@ process_installtype() {
 	elif [ "$install_type" = 'SEARCHNODE' ]; then
 		is_searchnode=true
 	elif [ "$install_type" = 'HEAVYNODE' ]; then
-		is_heavy=true
+		is_heavynode=true
 	elif [ "$install_type" = 'FLEET' ]; then
 		is_fleet=true
 	elif [ "$install_type" = 'IDH' ]; then
@@ -2077,8 +2081,7 @@ secrets_pillar(){
 		"  playbook_admin: $PLAYBOOKADMINPASS"\
 		"  playbook_automation: $PLAYBOOKAUTOMATIONPASS"\
 		"  playbook_automation_api_key: "\
-		"  influx_pass: $INFLUXPASS"\
+		"  influx_pass: $INFLUXPASS" > $local_salt_dir/pillar/secrets.sls
-		"  influx_token: $INFLUXTOKEN" > $local_salt_dir/pillar/secrets.sls
   fi
 }
 

setup/so-setup

@@ -658,5 +658,5 @@ if ! [[ -f $install_opt_file ]]; then
 		verify_setup
 	fi
 
-	# Need to make sure the latest install is located on the web server of the manager to check the versions and donwload the code if required
+	# Need to make sure the latest install is located on the web server of the manager to check the versions and download the code if required
 fi