Merge pull request #8123 from Security-Onion-Solutions/foxtrot

Merge foxtrot into dev

salt/common/tools/sbin/so-filebeat-module-setup

@@ -49,19 +49,18 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
 fi
 echo "Testing to see if the pipelines are already applied"
 ESVER=$({{ ELASTICCURL }} -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" |jq .version.number |tr -d \")
-PIPELINES=$({{ ELASTICCURL }} -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"/_ingest/pipeline/filebeat-$ESVER-suricata-eve-pipeline | jq . | wc -c)
+PIPELINES=$({{ ELASTICCURL }} -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"/_ingest/pipeline/filebeat-$ESVER-elasticsearch-server-pipeline | jq . | wc -c)
 
-if [[ "$PIPELINES" -lt 5 ]]; then
+if [[ "$PIPELINES" -lt 5 ]] || [ "$2" != "--force" ]; then
   echo "Setting up ingest pipeline(s)"
-
+{% from 'filebeat/modules.map.jinja' import MODULESMERGED with context %}
-  for MODULE in activemq apache auditd aws azure barracuda bluecoat cef checkpoint cisco coredns crowdstrike cyberark cylance elasticsearch envoyproxy f5 fortinet gcp google_workspace googlecloud gsuite haproxy ibmmq icinga iis imperva infoblox iptables juniper kafka kibana logstash microsoft mongodb mssql mysql nats netscout nginx o365 okta osquery panw postgresql rabbitmq radware redis santa snort snyk sonicwall sophos squid suricata system threatintel tomcat traefik zeek zscaler
+{%- for module in MODULESMERGED.modules.keys() %}
-  do
+  {%- for fileset in MODULESMERGED.modules[module] %}
-    echo "Loading $MODULE"
+      echo "{{ module }}.{{ fileset}}"
-    docker exec -i so-filebeat filebeat setup modules -pipelines -modules $MODULE -c $FB_MODULE_YML
+      docker exec -i so-filebeat filebeat setup --pipelines --modules {{ module }} -M "{{ module }}.{{ fileset }}.enabled=true" -c $FB_MODULE_YML
-    sleep 2
+      sleep 0.5
-  done
+  {% endfor %}
+{%- endfor %}
 else
   exit 0
 fi
-
-

salt/common/tools/sbin/so-user

@@ -238,7 +238,7 @@ function syncElastic() {
   syncElasticSystemUser "$authPillarJson" "so_monitor_user"  "$usersTmpFile"
 
   syncElasticSystemRole "$authPillarJson" "so_elastic_user"  "superuser" "$rolesTmpFile"
-  syncElasticSystemRole "$authPillarJson" "so_kibana_user"   "superuser" "$rolesTmpFile"
+  syncElasticSystemRole "$authPillarJson" "so_kibana_user"   "kibana_system" "$rolesTmpFile"
   syncElasticSystemRole "$authPillarJson" "so_logstash_user" "superuser" "$rolesTmpFile"
   syncElasticSystemRole "$authPillarJson" "so_beats_user"    "superuser" "$rolesTmpFile"
   syncElasticSystemRole "$authPillarJson" "so_monitor_user"  "remote_monitoring_collector" "$rolesTmpFile"

salt/common/tools/sbin/soup

@@ -422,8 +422,9 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == 2.3.80 ]] && up_to_2.3.90
     [[ "$INSTALLEDVERSION" == 2.3.90 || "$INSTALLEDVERSION" == 2.3.91 ]] && up_to_2.3.100
     [[ "$INSTALLEDVERSION" == 2.3.100 ]] && up_to_2.3.110
-    [[ "$INSTALLEDVERISON" == 2.3.110 ]] && up_to_2.3.120
+    [[ "$INSTALLEDVERSION" == 2.3.110 ]] && up_to_2.3.120
-    [[ "$INSTALLEDVERISON" == 2.3.120 ]] && up_to_2.3.130
+    [[ "$INSTALLEDVERSION" == 2.3.120 ]] && up_to_2.3.130
+    [[ "$INSTALLEDVERSION" == 2.3.130 ]] && up_to_2.3.140
     true
 }
 
@@ -780,6 +781,37 @@ up_to_2.3.130() {
   rm -f /opt/so/conf/navigator/layers/nav_layer_playbook.json
 }
 
+up_to_2.3.140() {
+  ## Deleting Elastalert indices to prevent issues with upgrade to Elastic 8 ##
+  echo "Deleting Elastalert indices to prevent issues with upgrade to Elastic 8..."
+  # Wait for ElasticSearch to initialize
+  echo -n "Waiting for ElasticSearch..."
+  COUNT=0
+  ELASTICSEARCH_CONNECTED="no" 
+  while [[ "$COUNT" -le 240 ]]; do
+    so-elasticsearch-query -k --output /dev/null
+      if [ $? -eq 0 ]; then
+        ELASTICSEARCH_CONNECTED="yes"
+        echo "connected!"
+          break
+      else
+        ((COUNT+=1))
+        sleep 1
+        echo -n "."
+      fi
+  done
+  if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
+    echo
+    echo -e "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'sudo docker ps' \n  -running 'sudo so-elastic-restart'"
+    echo
+    exit 1
+   fi
+   
+   # Delete Elastalert indices
+   for i in $(so-elasticsearch-query _cat/indices | grep elastalert | awk '{print $3}'); do so-elasticsearch-query $i -XDELETE; done
+   ##
+}
+
 verify_upgradespace() {
   CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//')
   if [ "$CURRENTSPACE" -lt "10" ]; then

salt/curator/files/bin/so-curator-close

@@ -23,8 +23,8 @@ read lastPID < $lf
 # if lastPID is not null and a process with that pid exists , exit
 [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
 echo $$ > $lf
-{% from 'filebeat/map.jinja' import THIRDPARTY with context %}
+
-{% from 'filebeat/map.jinja' import SO with context %}
+{% from 'filebeat/modules.map.jinja' import MODULESMERGED with context %}
 
 /usr/sbin/so-curator-closed-delete > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-close.yml > /dev/null 2>&1; 
@@ -36,9 +36,6 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-close.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-close.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-close.yml > /dev/null 2>&1;
-{% for INDEX in THIRDPARTY.modules.keys() -%}
+{% for INDEX in MODULESMERGED.modules.keys() -%}
-docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1;
-{% endfor -%}
-{% for INDEX in SO.modules.keys() -%}
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1{% if not loop.last %};{% endif %}
 {% endfor -%}

salt/curator/files/bin/so-curator-cluster-close

@@ -24,8 +24,7 @@ read lastPID < $lf
 [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
 echo $$ > $lf
 
-{% from 'filebeat/map.jinja' import THIRDPARTY with context %}
+{% from 'filebeat/modules.map.jinja' import MODULESMERGED with context %}
-{% from 'filebeat/map.jinja' import SO with context %}
 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-close.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-close.yml > /dev/null 2>&1; 
@@ -36,9 +35,6 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-close.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-close.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-close.yml > /dev/null 2>&1;
-{% for INDEX in THIRDPARTY.modules.keys() -%}
+{% for INDEX in MODULESMERGED.modules.keys() -%}
-docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1;
-{% endfor -%}
-{% for INDEX in SO.modules.keys() -%}
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1{% if not loop.last %};{% endif %}
 {% endfor -%}

salt/curator/files/bin/so-curator-cluster-delete

@@ -24,8 +24,7 @@ read lastPID < $lf
 [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
 echo $$ > $lf
 
-{% from 'filebeat/map.jinja' import THIRDPARTY with context %}
+{% from 'filebeat/modules.map.jinja' import MODULESMERGED with context %}
-{% from 'filebeat/map.jinja' import SO with context %}
 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-delete.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-delete.yml > /dev/null 2>&1; 
@@ -36,9 +35,6 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-delete.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-delete.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-delete.yml > /dev/null 2>&1;
-{% for INDEX in THIRDPARTY.modules.keys() -%}
+{% for INDEX in MODULESMERGED.modules.keys() -%}
-docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-delete.yml > /dev/null 2>&1;
-{% endfor -%}
-{% for INDEX in SO.modules.keys() -%}
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-delete.yml > /dev/null 2>&1{% if not loop.last %};{% endif %}
 {% endfor -%}

salt/curator/files/bin/so-curator-cluster-warm

@@ -24,8 +24,7 @@ read lastPID < $lf
 [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit
 echo $$ > $lf
 
-{% from 'filebeat/map.jinja' import THIRDPARTY with context %}
+{% from 'filebeat/modules.map.jinja' import MODULESMERGED with context %}
-{% from 'filebeat/map.jinja' import SO with context %}
 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-warm.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-warm.yml > /dev/null 2>&1; 
@@ -36,9 +35,6 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-warm.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-warm.yml > /dev/null 2>&1; 
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-warm.yml > /dev/null 2>&1;
-{% for INDEX in THIRDPARTY.modules.keys() -%}
+{% for INDEX in MODULESMERGED.modules.keys() -%}
-docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-warm.yml > /dev/null 2>&1;
-{% endfor -%}
-{% for INDEX in SO.modules.keys() -%}
 docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-warm.yml > /dev/null 2>&1{% if not loop.last %};{% endif %}
 {% endfor -%}

salt/elasticsearch/defaults.yaml

@@ -53,9 +53,6 @@ elasticsearch:
     script:
       max_compilations_rate: 20000/1m
     indices:
-      query:
-        bool:
-          max_clause_count: 3500
       id_field_data:
         enabled: false
     logger:

salt/filebeat/etc/filebeat.yml

@@ -64,6 +64,9 @@ logging.files:
   # automatically rotated
   rotateeverybytes: 10485760 # = 10MB
 
+  # Rotate on startup
+  rotateonstartup: false
+ 
   # Number of rotated log files to keep. Oldest files will be deleted first.
   keepfiles: 7
 
@@ -114,7 +117,7 @@ filebeat.inputs:
   fields_under_root: true
 
 {%- if grains['role'] in ['so-eval', 'so-standalone', 'so-manager', 'so-managersearch', 'so-import'] %}
-- type: log
+- type: filestream
   paths:
     - /logs/logscan/alerts.log
   fields:
@@ -131,7 +134,7 @@ filebeat.inputs:
 {%- if grains['role'] in ['so-eval', 'so-standalone', 'so-sensor',  'so-helix', 'so-heavynode', 'so-import'] %}
   {%- if ZEEKVER != 'SURICATA' %}
     {%- for LOGNAME in salt['pillar.get']('zeeklogs:enabled', '')  %}
-- type: log
+- type: filestream
   paths:
     - /nsm/zeek/logs/current/{{ LOGNAME }}.log
   fields:
@@ -146,7 +149,7 @@ filebeat.inputs:
   clean_removed: true
   close_removed: false
 
-- type: log
+- type: filestream
   paths:
     - /nsm/import/*/zeek/logs/{{ LOGNAME }}.log
   fields:
@@ -170,7 +173,7 @@ filebeat.inputs:
     {%- endfor %}
   {%- endif %}
 
-- type: log
+- type: filestream
   paths:
     - /nsm/suricata/eve*.json
   fields:
@@ -186,7 +189,7 @@ filebeat.inputs:
   clean_removed: false
   close_removed: false
 
-- type: log
+- type: filestream
   paths:
     - /nsm/import/*/suricata/eve*.json
   fields:
@@ -208,7 +211,7 @@ filebeat.inputs:
   clean_removed: false
   close_removed: false
   {%- if STRELKAENABLED == 1 %}
-- type: log
+- type: filestream
   paths:
     - /nsm/strelka/log/strelka.log
   fields:
@@ -229,7 +232,7 @@ filebeat.inputs:
 
 {%- if WAZUHENABLED == 1 %}
 
-- type: log
+- type: filestream
   paths:
     - /wazuh/archives/archives.json
   fields:
@@ -247,7 +250,7 @@ filebeat.inputs:
 
 {%- if FLEETMANAGER or FLEETNODE %}
 
-- type: log
+- type: filestream
   paths:
     - /nsm/osquery/fleet/result.log
   fields:
@@ -317,7 +320,7 @@ filebeat.inputs:
 {%- endif %}
 
 {%- if grains['role'] in ['so-eval', 'so-standalone', 'so-manager',  'so-managersearch', 'so-import'] %}
-- type: log
+- type: filestream
   paths:
     - /logs/kratos/kratos.log
   fields:
@@ -347,7 +350,7 @@ filebeat.inputs:
 {%- endif %}
 
 {%- if grains.role == 'so-idh' %}
-- type: log
+- type: filestream
   paths:
     - /nsm/idh/opencanary.log
   fields:

salt/filebeat/etc/module_config.yml.jinja

@@ -1,18 +1,2 @@
 # DO NOT EDIT THIS FILE
-{%- if MODULES.modules is iterable and MODULES.modules is not string and MODULES.modules|length > 0%}
+{{ MODULES|yaml(False) }}
-  {%- for module in MODULES.modules.keys() %}
-- module: {{ module }}
-    {%- for fileset in MODULES.modules[module] %}
-  {{ fileset }}:
-    enabled: {{ MODULES.modules[module][fileset].enabled|string|lower }}
-      {#- only manage the settings if the fileset is enabled #}
-      {%- if MODULES.modules[module][fileset].enabled %}
-        {%- for var, value in MODULES.modules[module][fileset].items() %}
-          {%- if var|lower != 'enabled' %}
-    {{ var }}: {{ value }}
-          {%- endif %}
-        {%- endfor %}
-      {%- endif %}
-    {%- endfor %}
-  {%- endfor %}
-{% endif %}

salt/filebeat/init.sls

@@ -18,8 +18,8 @@
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set LOCALHOSTNAME = salt['grains.get']('host') %}
 {% set MANAGER = salt['grains.get']('master') %}
-{% from 'filebeat/map.jinja' import THIRDPARTY with context %}
+{% from 'filebeat/modules.map.jinja' import MODULESMERGED with context %}
-{% from 'filebeat/map.jinja' import SO with context %}
+{% from 'filebeat/modules.map.jinja' import MODULESENABLED with context %}
 {% from 'filebeat/map.jinja' import FILEBEAT_EXTRA_HOSTS with context %}
 {% set ES_INCLUDED_NODES = ['so-eval', 'so-standalone', 'so-managersearch', 'so-node', 'so-heavynode', 'so-import'] %}
 
@@ -88,21 +88,21 @@ filebeatmoduleconf:
     - template: jinja
     - show_changes: False
 
-sodefaults_module_conf:
+merged_module_conf:
   file.managed:
-    - name: /opt/so/conf/filebeat/modules/securityonion.yml
+    - name: /opt/so/conf/filebeat/modules/modules.yml
     - source: salt://filebeat/etc/module_config.yml.jinja
     - template: jinja
     - defaults:
-        MODULES: {{ SO }}
+        MODULES: {{ MODULESENABLED }}
 
-thirdparty_module_conf:
+so_module_conf_remove:
-  file.managed:
+  file.absent:
+    - name: /opt/so/conf/filebeat/modules/securityonion.yml
+
+thirdyparty_module_conf_remove:
+  file.absent:
     - name: /opt/so/conf/filebeat/modules/thirdparty.yml
-    - source: salt://filebeat/etc/module_config.yml.jinja
-    - template: jinja
-    - defaults:
-        MODULES: {{ THIRDPARTY }}
     
 so-filebeat:
   docker_container.running:
@@ -127,14 +127,7 @@ so-filebeat:
         - 0.0.0.0:514:514/udp
         - 0.0.0.0:514:514/tcp
         - 0.0.0.0:5066:5066/tcp
-{% for module in THIRDPARTY.modules.keys() %}
+ 
-  {% for submodule in THIRDPARTY.modules[module] %}
-    {% if THIRDPARTY.modules[module][submodule].enabled and THIRDPARTY.modules[module][submodule]["var.syslog_port"] is defined %}
-        - {{ THIRDPARTY.modules[module][submodule].get("var.syslog_host", "0.0.0.0") }}:{{ THIRDPARTY.modules[module][submodule]["var.syslog_port"] }}:{{ THIRDPARTY.modules[module][submodule]["var.syslog_port"] }}/tcp
-        - {{ THIRDPARTY.modules[module][submodule].get("var.syslog_host", "0.0.0.0") }}:{{ THIRDPARTY.modules[module][submodule]["var.syslog_port"] }}:{{ THIRDPARTY.modules[module][submodule]["var.syslog_port"] }}/udp
-    {% endif %}
-  {% endfor %}
-{% endfor %}
     - watch:
       - file: filebeatconf
     - require:
@@ -144,7 +137,14 @@ so-filebeat:
       - x509: conf_filebeat_crt
       - x509: conf_filebeat_key
       - x509: trusttheca
-
+{% for module in MODULESMERGED.modules.keys() %}
+  {% for submodule in MODULESMERGED.modules[module] %}
+    {% if MODULESMERGED.modules[module][submodule].enabled and MODULESMERGED.modules[module][submodule]["var.syslog_port"] is defined %}
+        - {{ MODULESMERGED.modules[module][submodule].get("var.syslog_host", "0.0.0.0") }}:{{ MODULESMERGED.modules[module][submodule]["var.syslog_port"] }}:{{ MODULESMERGED.modules[module][submodule]["var.syslog_port"] }}/tcp
+        - {{ MODULESMERGED.modules[module][submodule].get("var.syslog_host", "0.0.0.0") }}:{{ MODULESMERGED.modules[module][submodule]["var.syslog_port"] }}:{{ MODULESMERGED.modules[module][submodule]["var.syslog_port"] }}/udp
+    {% endif %}
+  {% endfor %}
+{% endfor %}
 {% if grains.role in ES_INCLUDED_NODES %}
 run_module_setup:
   cmd.run:

salt/filebeat/map.jinja

@@ -1,10 +1,3 @@
-{% import_yaml 'filebeat/thirdpartydefaults.yaml' as TPDEFAULTS %}
-{% set THIRDPARTY = salt['pillar.get']('filebeat:third_party_filebeat', default=TPDEFAULTS.third_party_filebeat, merge=True) %}
-
-{% import_yaml 'filebeat/securityoniondefaults.yaml' as SODEFAULTS %}
-{% set SO = SODEFAULTS.securityonion_filebeat %}
-{#% set SO = salt['pillar.get']('filebeat:third_party_filebeat', default=SODEFAULTS.third_party_filebeat, merge=True) %#}
-
 {% set role = grains.role %}
 {% set FILEBEAT_EXTRA_HOSTS = [] %}
 {% set mainint = salt['pillar.get']('host:mainint') %}

salt/filebeat/modules.map.jinja

@@ -0,0 +1,18 @@
+{% import_yaml 'filebeat/thirdpartydefaults.yaml' as TPDEFAULTS %}
+{% import_yaml 'filebeat/securityoniondefaults.yaml' as SODEFAULTS %}
+{% set THIRDPARTY = salt['pillar.get']('filebeat:third_party_filebeat', default=TPDEFAULTS.third_party_filebeat, merge=True) %}
+{% set SO = salt['pillar.get']('filebeat:securityonion_filebeat', default=SODEFAULTS.securityonion_filebeat, merge=True) %}
+{% set MODULESMERGED = salt['defaults.merge'](SO, THIRDPARTY, in_place=False) %}
+
+{% set MODULESENABLED = [] %}
+{% for module in MODULESMERGED.modules.keys() %}
+  {% set ENABLEDFILESETS = {} %}                               
+  {% for fileset in MODULESMERGED.modules[module] %}                        
+    {% if MODULESMERGED.modules[module][fileset].get('enabled', False) %}  
+      {% do ENABLEDFILESETS.update({'module': module, fileset: MODULESMERGED.modules[module][fileset]}) %}
+    {% endif %}
+  {% endfor %}
+  {% if ENABLEDFILESETS|length > 0 %}
+    {% do MODULESENABLED.append(ENABLEDFILESETS) %}
+  {% endif %}
+{% endfor %}

salt/filebeat/thirdpartydefaults.yaml

@@ -74,12 +74,6 @@ third_party_filebeat:
         enabled: false
       amp:
         enabled: false
-    cyberark:
-      corepas:
-        enabled: false
-        var.input: udp
-        var.syslog_host: 0.0.0.0
-        var.syslog_port: 9527
     cylance:
       protect:
         enabled: false
@@ -259,8 +253,6 @@ third_party_filebeat:
         enabled: false
       anomalithreatstream:
         enabled: false
-      recordedfuture:
-        enabled: false
     zscaler:
       zia:
         enabled: false 

salt/kibana/bin/so-kibana-config-load

@@ -59,7 +59,7 @@ update() {
 
     IFS=$'\r\n' GLOBIGNORE='*' command eval  'LINES=($(cat $1))'
     for i in "${LINES[@]}"; do
-      RESPONSE=$({{ ELASTICCURL }} -X PUT "localhost:5601/api/saved_objects/config/7.17.4" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ")
+      RESPONSE=$({{ ELASTICCURL }} -X PUT "localhost:5601/api/saved_objects/config/8.2.2" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ")
       echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi
     done
  

salt/kibana/config.map.jinja

@@ -2,7 +2,7 @@
 {% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %}
 
 {% if salt['pillar.get']('elasticsearch:auth:enabled', False) %}
-  {% do KIBANACONFIG.kibana.config.elasticsearch.update({'username': salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user'), 'password': salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass')}) %}
+  {% do KIBANACONFIG.kibana.config.elasticsearch.update({'username': salt['pillar.get']('elasticsearch:auth:users:so_kibana_user:user'), 'password': salt['pillar.get']('elasticsearch:auth:users:so_kibana_user:pass')}) %}
 {% else %}
   {% do KIBANACONFIG.kibana.config.xpack.update({'security': {'authc': {'providers': {'anonymous': {'anonymous1': {'order': 0, 'credentials': 'elasticsearch_anonymous_user'}}}}}}) %}
 {% endif %}

salt/kibana/defaults.yaml

@@ -28,7 +28,8 @@ kibana:
     security:
       showInsecureClusterWarning: False
     xpack:
-      ml:
-        enabled: False
       security:
-        secureCookies: True
+        secureCookies: true
+      reporting:
+        kibanaServer:
+          hostname: localhost

salt/kibana/files/config_saved_objects.ndjson

@@ -1 +1 @@
-{"attributes": {"buildNum": 39457,"defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n  \"from\": \"now-24h\",\n  \"to\": \"now\"\n}"},"coreMigrationVersion": "7.17.4","id": "7.17.4","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="}
+{"attributes": {"buildNum": 39457,"defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n  \"from\": \"now-24h\",\n  \"to\": \"now\"\n}"},"coreMigrationVersion": "8.2.2","id": "8.2.2","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="}

salt/utility/bin/crossthestreams

@@ -30,13 +30,13 @@ fi
 echo "Applying cross cluster search config..."
     {{ ELASTICCURL }} -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings \
          -H 'Content-Type: application/json' \
-         -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ MANAGER }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}"
+         -d "{\"persistent\": {\"cluster\": {\"remote\": {\"{{ MANAGER }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}"
 
 # Add all the search nodes to cross cluster searching.
 {%- if TRUECLUSTER is sameas false %}
   {%- if salt['pillar.get']('nodestab', {}) %}
     {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %}
-{{ ELASTICCURL }} -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}'
+{{ ELASTICCURL }} -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"cluster": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}'
     {%- endfor %}
   {%- endif %}
 {%- endif %}

salt/utility/bin/eval

@@ -28,4 +28,4 @@ fi
 echo "Applying cross cluster search config..."
     {{ ELASTICCURL }} -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings \
          -H 'Content-Type: application/json' \
-         -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ grains.host }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}"
+         -d "{\"persistent\": {\"cluster\": {\"remote\": {\"{{ grains.host }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}"