diff --git a/salt/common/tools/sbin/so-filebeat-module-setup b/salt/common/tools/sbin/so-filebeat-module-setup index bb6ff86a6..945c3c58a 100755 --- a/salt/common/tools/sbin/so-filebeat-module-setup +++ b/salt/common/tools/sbin/so-filebeat-module-setup @@ -49,19 +49,18 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then fi echo "Testing to see if the pipelines are already applied" ESVER=$({{ ELASTICCURL }} -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" |jq .version.number |tr -d \") -PIPELINES=$({{ ELASTICCURL }} -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"/_ingest/pipeline/filebeat-$ESVER-suricata-eve-pipeline | jq . | wc -c) +PIPELINES=$({{ ELASTICCURL }} -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"/_ingest/pipeline/filebeat-$ESVER-elasticsearch-server-pipeline | jq . | wc -c) -if [[ "$PIPELINES" -lt 5 ]]; then +if [[ "$PIPELINES" -lt 5 ]] || [ "$2" != "--force" ]; then echo "Setting up ingest pipeline(s)" - - for MODULE in activemq apache auditd aws azure barracuda bluecoat cef checkpoint cisco coredns crowdstrike cyberark cylance elasticsearch envoyproxy f5 fortinet gcp google_workspace googlecloud gsuite haproxy ibmmq icinga iis imperva infoblox iptables juniper kafka kibana logstash microsoft mongodb mssql mysql nats netscout nginx o365 okta osquery panw postgresql rabbitmq radware redis santa snort snyk sonicwall sophos squid suricata system threatintel tomcat traefik zeek zscaler - do - echo "Loading $MODULE" - docker exec -i so-filebeat filebeat setup modules -pipelines -modules $MODULE -c $FB_MODULE_YML - sleep 2 - done +{% from 'filebeat/modules.map.jinja' import MODULESMERGED with context %} +{%- for module in MODULESMERGED.modules.keys() %} + {%- for fileset in MODULESMERGED.modules[module] %} + echo "{{ module }}.{{ fileset}}" + docker exec -i so-filebeat filebeat setup --pipelines --modules {{ module }} -M "{{ module }}.{{ fileset }}.enabled=true" -c $FB_MODULE_YML + sleep 0.5 + {% endfor %} +{%- endfor %} else exit 0 fi - - diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 30162e4a9..05a44a19d 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -238,7 +238,7 @@ function syncElastic() { syncElasticSystemUser "$authPillarJson" "so_monitor_user" "$usersTmpFile" syncElasticSystemRole "$authPillarJson" "so_elastic_user" "superuser" "$rolesTmpFile" - syncElasticSystemRole "$authPillarJson" "so_kibana_user" "superuser" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_kibana_user" "kibana_system" "$rolesTmpFile" syncElasticSystemRole "$authPillarJson" "so_logstash_user" "superuser" "$rolesTmpFile" syncElasticSystemRole "$authPillarJson" "so_beats_user" "superuser" "$rolesTmpFile" syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_collector" "$rolesTmpFile" diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 2e76bd9ef..592c11d98 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -422,8 +422,9 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.3.80 ]] && up_to_2.3.90 [[ "$INSTALLEDVERSION" == 2.3.90 || "$INSTALLEDVERSION" == 2.3.91 ]] && up_to_2.3.100 [[ "$INSTALLEDVERSION" == 2.3.100 ]] && up_to_2.3.110 - [[ "$INSTALLEDVERISON" == 2.3.110 ]] && up_to_2.3.120 - [[ "$INSTALLEDVERISON" == 2.3.120 ]] && up_to_2.3.130 + [[ "$INSTALLEDVERSION" == 2.3.110 ]] && up_to_2.3.120 + [[ "$INSTALLEDVERSION" == 2.3.120 ]] && up_to_2.3.130 + [[ "$INSTALLEDVERSION" == 2.3.130 ]] && up_to_2.3.140 true } @@ -780,6 +781,37 @@ up_to_2.3.130() { rm -f /opt/so/conf/navigator/layers/nav_layer_playbook.json } +up_to_2.3.140() { + ## Deleting Elastalert indices to prevent issues with upgrade to Elastic 8 ## + echo "Deleting Elastalert indices to prevent issues with upgrade to Elastic 8..." + # Wait for ElasticSearch to initialize + echo -n "Waiting for ElasticSearch..." + COUNT=0 + ELASTICSEARCH_CONNECTED="no" + while [[ "$COUNT" -le 240 ]]; do + so-elasticsearch-query -k --output /dev/null + if [ $? -eq 0 ]; then + ELASTICSEARCH_CONNECTED="yes" + echo "connected!" + break + else + ((COUNT+=1)) + sleep 1 + echo -n "." + fi + done + if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then + echo + echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" + echo + exit 1 + fi + + # Delete Elastalert indices + for i in $(so-elasticsearch-query _cat/indices | grep elastalert | awk '{print $3}'); do so-elasticsearch-query $i -XDELETE; done + ## +} + verify_upgradespace() { CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//') if [ "$CURRENTSPACE" -lt "10" ]; then diff --git a/salt/curator/files/bin/so-curator-close b/salt/curator/files/bin/so-curator-close index f445d7e6b..25a19c671 100644 --- a/salt/curator/files/bin/so-curator-close +++ b/salt/curator/files/bin/so-curator-close @@ -23,8 +23,8 @@ read lastPID < $lf # if lastPID is not null and a process with that pid exists , exit [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit echo $$ > $lf -{% from 'filebeat/map.jinja' import THIRDPARTY with context %} -{% from 'filebeat/map.jinja' import SO with context %} + +{% from 'filebeat/modules.map.jinja' import MODULESMERGED with context %} /usr/sbin/so-curator-closed-delete > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-close.yml > /dev/null 2>&1; @@ -36,9 +36,6 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-close.yml > /dev/null 2>&1; -{% for INDEX in THIRDPARTY.modules.keys() -%} -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1; -{% endfor -%} -{% for INDEX in SO.modules.keys() -%} +{% for INDEX in MODULESMERGED.modules.keys() -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} {% endfor -%} diff --git a/salt/curator/files/bin/so-curator-cluster-close b/salt/curator/files/bin/so-curator-cluster-close index 833cef335..0da245516 100644 --- a/salt/curator/files/bin/so-curator-cluster-close +++ b/salt/curator/files/bin/so-curator-cluster-close @@ -24,8 +24,7 @@ read lastPID < $lf [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit echo $$ > $lf -{% from 'filebeat/map.jinja' import THIRDPARTY with context %} -{% from 'filebeat/map.jinja' import SO with context %} +{% from 'filebeat/modules.map.jinja' import MODULESMERGED with context %} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-close.yml > /dev/null 2>&1; @@ -36,9 +35,6 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-close.yml > /dev/null 2>&1; -{% for INDEX in THIRDPARTY.modules.keys() -%} -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1; -{% endfor -%} -{% for INDEX in SO.modules.keys() -%} +{% for INDEX in MODULESMERGED.modules.keys() -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} {% endfor -%} diff --git a/salt/curator/files/bin/so-curator-cluster-delete b/salt/curator/files/bin/so-curator-cluster-delete index e70c4eb02..313a0c311 100644 --- a/salt/curator/files/bin/so-curator-cluster-delete +++ b/salt/curator/files/bin/so-curator-cluster-delete @@ -24,8 +24,7 @@ read lastPID < $lf [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit echo $$ > $lf -{% from 'filebeat/map.jinja' import THIRDPARTY with context %} -{% from 'filebeat/map.jinja' import SO with context %} +{% from 'filebeat/modules.map.jinja' import MODULESMERGED with context %} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-delete.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-delete.yml > /dev/null 2>&1; @@ -36,9 +35,6 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-delete.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-delete.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-delete.yml > /dev/null 2>&1; -{% for INDEX in THIRDPARTY.modules.keys() -%} -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-delete.yml > /dev/null 2>&1; -{% endfor -%} -{% for INDEX in SO.modules.keys() -%} +{% for INDEX in MODULESMERGED.modules.keys() -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-delete.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} {% endfor -%} diff --git a/salt/curator/files/bin/so-curator-cluster-warm b/salt/curator/files/bin/so-curator-cluster-warm index 7279c6d41..1a6791bac 100644 --- a/salt/curator/files/bin/so-curator-cluster-warm +++ b/salt/curator/files/bin/so-curator-cluster-warm @@ -24,8 +24,7 @@ read lastPID < $lf [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit echo $$ > $lf -{% from 'filebeat/map.jinja' import THIRDPARTY with context %} -{% from 'filebeat/map.jinja' import SO with context %} +{% from 'filebeat/modules.map.jinja' import MODULESMERGED with context %} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-warm.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-warm.yml > /dev/null 2>&1; @@ -36,9 +35,6 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-warm.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-warm.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-warm.yml > /dev/null 2>&1; -{% for INDEX in THIRDPARTY.modules.keys() -%} -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-warm.yml > /dev/null 2>&1; -{% endfor -%} -{% for INDEX in SO.modules.keys() -%} +{% for INDEX in MODULESMERGED.modules.keys() -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-warm.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} {% endfor -%} diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 03fc3facf..830d1372c 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -53,9 +53,6 @@ elasticsearch: script: max_compilations_rate: 20000/1m indices: - query: - bool: - max_clause_count: 3500 id_field_data: enabled: false logger: diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 7bfc17d97..04a3351a3 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -64,6 +64,9 @@ logging.files: # automatically rotated rotateeverybytes: 10485760 # = 10MB + # Rotate on startup + rotateonstartup: false + # Number of rotated log files to keep. Oldest files will be deleted first. keepfiles: 7 @@ -114,7 +117,7 @@ filebeat.inputs: fields_under_root: true {%- if grains['role'] in ['so-eval', 'so-standalone', 'so-manager', 'so-managersearch', 'so-import'] %} -- type: log +- type: filestream paths: - /logs/logscan/alerts.log fields: @@ -131,7 +134,7 @@ filebeat.inputs: {%- if grains['role'] in ['so-eval', 'so-standalone', 'so-sensor', 'so-helix', 'so-heavynode', 'so-import'] %} {%- if ZEEKVER != 'SURICATA' %} {%- for LOGNAME in salt['pillar.get']('zeeklogs:enabled', '') %} -- type: log +- type: filestream paths: - /nsm/zeek/logs/current/{{ LOGNAME }}.log fields: @@ -146,7 +149,7 @@ filebeat.inputs: clean_removed: true close_removed: false -- type: log +- type: filestream paths: - /nsm/import/*/zeek/logs/{{ LOGNAME }}.log fields: @@ -170,7 +173,7 @@ filebeat.inputs: {%- endfor %} {%- endif %} -- type: log +- type: filestream paths: - /nsm/suricata/eve*.json fields: @@ -186,7 +189,7 @@ filebeat.inputs: clean_removed: false close_removed: false -- type: log +- type: filestream paths: - /nsm/import/*/suricata/eve*.json fields: @@ -208,7 +211,7 @@ filebeat.inputs: clean_removed: false close_removed: false {%- if STRELKAENABLED == 1 %} -- type: log +- type: filestream paths: - /nsm/strelka/log/strelka.log fields: @@ -229,7 +232,7 @@ filebeat.inputs: {%- if WAZUHENABLED == 1 %} -- type: log +- type: filestream paths: - /wazuh/archives/archives.json fields: @@ -247,7 +250,7 @@ filebeat.inputs: {%- if FLEETMANAGER or FLEETNODE %} -- type: log +- type: filestream paths: - /nsm/osquery/fleet/result.log fields: @@ -317,7 +320,7 @@ filebeat.inputs: {%- endif %} {%- if grains['role'] in ['so-eval', 'so-standalone', 'so-manager', 'so-managersearch', 'so-import'] %} -- type: log +- type: filestream paths: - /logs/kratos/kratos.log fields: @@ -347,7 +350,7 @@ filebeat.inputs: {%- endif %} {%- if grains.role == 'so-idh' %} -- type: log +- type: filestream paths: - /nsm/idh/opencanary.log fields: diff --git a/salt/filebeat/etc/module_config.yml.jinja b/salt/filebeat/etc/module_config.yml.jinja index 733d47c7e..5d8782c01 100644 --- a/salt/filebeat/etc/module_config.yml.jinja +++ b/salt/filebeat/etc/module_config.yml.jinja @@ -1,18 +1,2 @@ # DO NOT EDIT THIS FILE -{%- if MODULES.modules is iterable and MODULES.modules is not string and MODULES.modules|length > 0%} - {%- for module in MODULES.modules.keys() %} -- module: {{ module }} - {%- for fileset in MODULES.modules[module] %} - {{ fileset }}: - enabled: {{ MODULES.modules[module][fileset].enabled|string|lower }} - {#- only manage the settings if the fileset is enabled #} - {%- if MODULES.modules[module][fileset].enabled %} - {%- for var, value in MODULES.modules[module][fileset].items() %} - {%- if var|lower != 'enabled' %} - {{ var }}: {{ value }} - {%- endif %} - {%- endfor %} - {%- endif %} - {%- endfor %} - {%- endfor %} -{% endif %} +{{ MODULES|yaml(False) }} diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 82622c4b2..ea04c0311 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -18,8 +18,8 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set LOCALHOSTNAME = salt['grains.get']('host') %} {% set MANAGER = salt['grains.get']('master') %} -{% from 'filebeat/map.jinja' import THIRDPARTY with context %} -{% from 'filebeat/map.jinja' import SO with context %} +{% from 'filebeat/modules.map.jinja' import MODULESMERGED with context %} +{% from 'filebeat/modules.map.jinja' import MODULESENABLED with context %} {% from 'filebeat/map.jinja' import FILEBEAT_EXTRA_HOSTS with context %} {% set ES_INCLUDED_NODES = ['so-eval', 'so-standalone', 'so-managersearch', 'so-node', 'so-heavynode', 'so-import'] %} @@ -88,21 +88,21 @@ filebeatmoduleconf: - template: jinja - show_changes: False -sodefaults_module_conf: +merged_module_conf: file.managed: - - name: /opt/so/conf/filebeat/modules/securityonion.yml + - name: /opt/so/conf/filebeat/modules/modules.yml - source: salt://filebeat/etc/module_config.yml.jinja - template: jinja - defaults: - MODULES: {{ SO }} + MODULES: {{ MODULESENABLED }} -thirdparty_module_conf: - file.managed: +so_module_conf_remove: + file.absent: + - name: /opt/so/conf/filebeat/modules/securityonion.yml + +thirdyparty_module_conf_remove: + file.absent: - name: /opt/so/conf/filebeat/modules/thirdparty.yml - - source: salt://filebeat/etc/module_config.yml.jinja - - template: jinja - - defaults: - MODULES: {{ THIRDPARTY }} so-filebeat: docker_container.running: @@ -127,14 +127,7 @@ so-filebeat: - 0.0.0.0:514:514/udp - 0.0.0.0:514:514/tcp - 0.0.0.0:5066:5066/tcp -{% for module in THIRDPARTY.modules.keys() %} - {% for submodule in THIRDPARTY.modules[module] %} - {% if THIRDPARTY.modules[module][submodule].enabled and THIRDPARTY.modules[module][submodule]["var.syslog_port"] is defined %} - - {{ THIRDPARTY.modules[module][submodule].get("var.syslog_host", "0.0.0.0") }}:{{ THIRDPARTY.modules[module][submodule]["var.syslog_port"] }}:{{ THIRDPARTY.modules[module][submodule]["var.syslog_port"] }}/tcp - - {{ THIRDPARTY.modules[module][submodule].get("var.syslog_host", "0.0.0.0") }}:{{ THIRDPARTY.modules[module][submodule]["var.syslog_port"] }}:{{ THIRDPARTY.modules[module][submodule]["var.syslog_port"] }}/udp - {% endif %} - {% endfor %} -{% endfor %} + - watch: - file: filebeatconf - require: @@ -144,7 +137,14 @@ so-filebeat: - x509: conf_filebeat_crt - x509: conf_filebeat_key - x509: trusttheca - +{% for module in MODULESMERGED.modules.keys() %} + {% for submodule in MODULESMERGED.modules[module] %} + {% if MODULESMERGED.modules[module][submodule].enabled and MODULESMERGED.modules[module][submodule]["var.syslog_port"] is defined %} + - {{ MODULESMERGED.modules[module][submodule].get("var.syslog_host", "0.0.0.0") }}:{{ MODULESMERGED.modules[module][submodule]["var.syslog_port"] }}:{{ MODULESMERGED.modules[module][submodule]["var.syslog_port"] }}/tcp + - {{ MODULESMERGED.modules[module][submodule].get("var.syslog_host", "0.0.0.0") }}:{{ MODULESMERGED.modules[module][submodule]["var.syslog_port"] }}:{{ MODULESMERGED.modules[module][submodule]["var.syslog_port"] }}/udp + {% endif %} + {% endfor %} +{% endfor %} {% if grains.role in ES_INCLUDED_NODES %} run_module_setup: cmd.run: diff --git a/salt/filebeat/map.jinja b/salt/filebeat/map.jinja index f12714176..a93eedce0 100644 --- a/salt/filebeat/map.jinja +++ b/salt/filebeat/map.jinja @@ -1,10 +1,3 @@ -{% import_yaml 'filebeat/thirdpartydefaults.yaml' as TPDEFAULTS %} -{% set THIRDPARTY = salt['pillar.get']('filebeat:third_party_filebeat', default=TPDEFAULTS.third_party_filebeat, merge=True) %} - -{% import_yaml 'filebeat/securityoniondefaults.yaml' as SODEFAULTS %} -{% set SO = SODEFAULTS.securityonion_filebeat %} -{#% set SO = salt['pillar.get']('filebeat:third_party_filebeat', default=SODEFAULTS.third_party_filebeat, merge=True) %#} - {% set role = grains.role %} {% set FILEBEAT_EXTRA_HOSTS = [] %} {% set mainint = salt['pillar.get']('host:mainint') %} diff --git a/salt/filebeat/modules.map.jinja b/salt/filebeat/modules.map.jinja new file mode 100644 index 000000000..c4a0a6db7 --- /dev/null +++ b/salt/filebeat/modules.map.jinja @@ -0,0 +1,18 @@ +{% import_yaml 'filebeat/thirdpartydefaults.yaml' as TPDEFAULTS %} +{% import_yaml 'filebeat/securityoniondefaults.yaml' as SODEFAULTS %} +{% set THIRDPARTY = salt['pillar.get']('filebeat:third_party_filebeat', default=TPDEFAULTS.third_party_filebeat, merge=True) %} +{% set SO = salt['pillar.get']('filebeat:securityonion_filebeat', default=SODEFAULTS.securityonion_filebeat, merge=True) %} +{% set MODULESMERGED = salt['defaults.merge'](SO, THIRDPARTY, in_place=False) %} + +{% set MODULESENABLED = [] %} +{% for module in MODULESMERGED.modules.keys() %} + {% set ENABLEDFILESETS = {} %} + {% for fileset in MODULESMERGED.modules[module] %} + {% if MODULESMERGED.modules[module][fileset].get('enabled', False) %} + {% do ENABLEDFILESETS.update({'module': module, fileset: MODULESMERGED.modules[module][fileset]}) %} + {% endif %} + {% endfor %} + {% if ENABLEDFILESETS|length > 0 %} + {% do MODULESENABLED.append(ENABLEDFILESETS) %} + {% endif %} +{% endfor %} diff --git a/salt/filebeat/thirdpartydefaults.yaml b/salt/filebeat/thirdpartydefaults.yaml index 3be8bb901..48411cad9 100644 --- a/salt/filebeat/thirdpartydefaults.yaml +++ b/salt/filebeat/thirdpartydefaults.yaml @@ -74,12 +74,6 @@ third_party_filebeat: enabled: false amp: enabled: false - cyberark: - corepas: - enabled: false - var.input: udp - var.syslog_host: 0.0.0.0 - var.syslog_port: 9527 cylance: protect: enabled: false @@ -259,8 +253,6 @@ third_party_filebeat: enabled: false anomalithreatstream: enabled: false - recordedfuture: - enabled: false zscaler: zia: enabled: false diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load index 73c2d9a0f..dcb66355f 100644 --- a/salt/kibana/bin/so-kibana-config-load +++ b/salt/kibana/bin/so-kibana-config-load @@ -59,7 +59,7 @@ update() { IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' for i in "${LINES[@]}"; do - RESPONSE=$({{ ELASTICCURL }} -X PUT "localhost:5601/api/saved_objects/config/7.17.4" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") + RESPONSE=$({{ ELASTICCURL }} -X PUT "localhost:5601/api/saved_objects/config/8.2.2" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi done diff --git a/salt/kibana/config.map.jinja b/salt/kibana/config.map.jinja index d595092de..cadfcab4e 100644 --- a/salt/kibana/config.map.jinja +++ b/salt/kibana/config.map.jinja @@ -2,7 +2,7 @@ {% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %} {% if salt['pillar.get']('elasticsearch:auth:enabled', False) %} - {% do KIBANACONFIG.kibana.config.elasticsearch.update({'username': salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user'), 'password': salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass')}) %} + {% do KIBANACONFIG.kibana.config.elasticsearch.update({'username': salt['pillar.get']('elasticsearch:auth:users:so_kibana_user:user'), 'password': salt['pillar.get']('elasticsearch:auth:users:so_kibana_user:pass')}) %} {% else %} {% do KIBANACONFIG.kibana.config.xpack.update({'security': {'authc': {'providers': {'anonymous': {'anonymous1': {'order': 0, 'credentials': 'elasticsearch_anonymous_user'}}}}}}) %} {% endif %} diff --git a/salt/kibana/defaults.yaml b/salt/kibana/defaults.yaml index eae767386..de78fc12c 100644 --- a/salt/kibana/defaults.yaml +++ b/salt/kibana/defaults.yaml @@ -28,7 +28,8 @@ kibana: security: showInsecureClusterWarning: False xpack: - ml: - enabled: False security: - secureCookies: True + secureCookies: true + reporting: + kibanaServer: + hostname: localhost diff --git a/salt/kibana/files/config_saved_objects.ndjson b/salt/kibana/files/config_saved_objects.ndjson index 44a1d0157..435cedfe0 100644 --- a/salt/kibana/files/config_saved_objects.ndjson +++ b/salt/kibana/files/config_saved_objects.ndjson @@ -1 +1 @@ -{"attributes": {"buildNum": 39457,"defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "7.17.4","id": "7.17.4","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} +{"attributes": {"buildNum": 39457,"defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.2.2","id": "8.2.2","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} diff --git a/salt/utility/bin/crossthestreams b/salt/utility/bin/crossthestreams index 0b2d17918..38222bbec 100644 --- a/salt/utility/bin/crossthestreams +++ b/salt/utility/bin/crossthestreams @@ -30,13 +30,13 @@ fi echo "Applying cross cluster search config..." {{ ELASTICCURL }} -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings \ -H 'Content-Type: application/json' \ - -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ MANAGER }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}" + -d "{\"persistent\": {\"cluster\": {\"remote\": {\"{{ MANAGER }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}" # Add all the search nodes to cross cluster searching. {%- if TRUECLUSTER is sameas false %} {%- if salt['pillar.get']('nodestab', {}) %} {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} -{{ ELASTICCURL }} -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}' +{{ ELASTICCURL }} -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"cluster": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}' {%- endfor %} {%- endif %} {%- endif %} diff --git a/salt/utility/bin/eval b/salt/utility/bin/eval index eba0df039..e96fc9a78 100644 --- a/salt/utility/bin/eval +++ b/salt/utility/bin/eval @@ -28,4 +28,4 @@ fi echo "Applying cross cluster search config..." {{ ELASTICCURL }} -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings \ -H 'Content-Type: application/json' \ - -d "{\"persistent\": {\"search\": {\"remote\": {\"{{ grains.host }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}" + -d "{\"persistent\": {\"cluster\": {\"remote\": {\"{{ grains.host }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}"