diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 1fac0f0e3..3a6aa2f6a 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -33,6 +33,8 @@ 'kratos', 'hydra', 'elasticfleet', + 'elasticfleet.manager', + 'elasticsearch.cluster', 'elastic-fleet-package-registry', 'utility' ] %} diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index 89ba1f80a..cb189f9a9 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -17,65 +17,17 @@ include: - logstash.ssl - elasticfleet.config - elasticfleet.sostatus +{%- if GLOBALS.role != "so-fleet" %} + - elasticfleet.manager +{%- endif %} -{% if grains.role not in ['so-fleet'] %} +{% if GLOBALS.role != "so-fleet" %} # Wait for Elasticsearch to be ready - no reason to try running Elastic Fleet server if ES is not ready wait_for_elasticsearch_elasticfleet: cmd.run: - name: so-elasticsearch-wait -{% endif %} - -# If enabled, automatically update Fleet Logstash Outputs -{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval', 'so-fleet'] %} -so-elastic-fleet-auto-configure-logstash-outputs: - cmd.run: - - name: /usr/sbin/so-elastic-fleet-outputs-update - - retry: - attempts: 4 - interval: 30 - -{# Separate from above in order to catch elasticfleet-logstash.crt changes and force update to fleet output policy #} -so-elastic-fleet-auto-configure-logstash-outputs-force: - cmd.run: - - name: /usr/sbin/so-elastic-fleet-outputs-update --certs - - retry: - attempts: 4 - interval: 30 - - onchanges: - - x509: etc_elasticfleet_logstash_crt - - x509: elasticfleet_kafka_crt -{% endif %} - -# If enabled, automatically update Fleet Server URLs & ES Connection -{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-fleet'] %} -so-elastic-fleet-auto-configure-server-urls: - cmd.run: - - name: /usr/sbin/so-elastic-fleet-urls-update - - retry: - attempts: 4 - interval: 30 -{% endif %} - -# Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs -{% if grains.role not in ['so-fleet'] %} -so-elastic-fleet-auto-configure-elasticsearch-urls: - cmd.run: - - name: /usr/sbin/so-elastic-fleet-es-url-update - - retry: - attempts: 4 - interval: 30 - -so-elastic-fleet-auto-configure-artifact-urls: - cmd.run: - - name: /usr/sbin/so-elastic-fleet-artifacts-url-update - - retry: - attempts: 4 - interval: 30 - -{% endif %} # Sync Elastic Agent artifacts to Fleet Node -{% if grains.role in ['so-fleet'] %} elasticagent_syncartifacts: file.recurse: - name: /nsm/elastic-fleet/artifacts/beats @@ -149,57 +101,6 @@ so-elastic-fleet: - x509: etc_elasticfleet_crt {% endif %} -{% if GLOBALS.role != "so-fleet" %} -so-elastic-fleet-package-statefile: - file.managed: - - name: /opt/so/state/elastic_fleet_packages.txt - - contents: {{ELASTICFLEETMERGED.packages}} - -so-elastic-fleet-package-upgrade: - cmd.run: - - name: /usr/sbin/so-elastic-fleet-package-upgrade - - retry: - attempts: 3 - interval: 10 - - onchanges: - - file: /opt/so/state/elastic_fleet_packages.txt - -so-elastic-fleet-integrations: - cmd.run: - - name: /usr/sbin/so-elastic-fleet-integration-policy-load - - retry: - attempts: 3 - interval: 10 - -so-elastic-agent-grid-upgrade: - cmd.run: - - name: /usr/sbin/so-elastic-agent-grid-upgrade - - retry: - attempts: 12 - interval: 5 - -so-elastic-fleet-integration-upgrade: - cmd.run: - - name: /usr/sbin/so-elastic-fleet-integration-upgrade - - retry: - attempts: 3 - interval: 10 - -{# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #} -so-elastic-fleet-addon-integrations: - cmd.run: - - name: /usr/sbin/so-elastic-fleet-optional-integrations-load - -{% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %} -so-elastic-defend-manage-filters-file-watch: - cmd.run: - - name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log - - onchanges: - - file: elasticdefendcustom - - file: elasticdefenddisabled -{% endif %} -{% endif %} - delete_so-elastic-fleet_so-status.disabled: file.uncomment: - name: /opt/so/conf/so-status/so-status.conf diff --git a/salt/elasticfleet/manager.sls b/salt/elasticfleet/manager.sls new file mode 100644 index 000000000..00fead9cf --- /dev/null +++ b/salt/elasticfleet/manager.sls @@ -0,0 +1,112 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls in allowed_states %} +{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %} + +include: + - elasticfleet.config + +# If enabled, automatically update Fleet Logstash Outputs +{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval'] %} +so-elastic-fleet-auto-configure-logstash-outputs: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-outputs-update + - retry: + attempts: 4 + interval: 30 + +{# Separate from above in order to catch elasticfleet-logstash.crt changes and force update to fleet output policy #} +so-elastic-fleet-auto-configure-logstash-outputs-force: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-outputs-update --certs + - retry: + attempts: 4 + interval: 30 + - onchanges: + - x509: etc_elasticfleet_logstash_crt + - x509: elasticfleet_kafka_crt +{% endif %} + +# If enabled, automatically update Fleet Server URLs & ES Connection +so-elastic-fleet-auto-configure-server-urls: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-urls-update + - retry: + attempts: 4 + interval: 30 + +# Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs +so-elastic-fleet-auto-configure-elasticsearch-urls: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-es-url-update + - retry: + attempts: 4 + interval: 30 + +so-elastic-fleet-auto-configure-artifact-urls: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-artifacts-url-update + - retry: + attempts: 4 + interval: 30 + +so-elastic-fleet-package-statefile: + file.managed: + - name: /opt/so/state/elastic_fleet_packages.txt + - contents: {{ELASTICFLEETMERGED.packages}} + +so-elastic-fleet-package-upgrade: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-package-upgrade + - retry: + attempts: 3 + interval: 10 + - onchanges: + - file: /opt/so/state/elastic_fleet_packages.txt + +so-elastic-fleet-integrations: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-integration-policy-load + - retry: + attempts: 3 + interval: 10 + +so-elastic-agent-grid-upgrade: + cmd.run: + - name: /usr/sbin/so-elastic-agent-grid-upgrade + - retry: + attempts: 12 + interval: 5 + +so-elastic-fleet-integration-upgrade: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-integration-upgrade + - retry: + attempts: 3 + interval: 10 + +{# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #} +so-elastic-fleet-addon-integrations: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-optional-integrations-load + +{% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %} +so-elastic-defend-manage-filters-file-watch: + cmd.run: + - name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log + - onchanges: + - file: elasticdefendcustom + - file: elasticdefenddisabled +{% endif %} + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/elasticsearch/cluster.sls b/salt/elasticsearch/cluster.sls index 7a8a6675c..e25aed36a 100644 --- a/salt/elasticsearch/cluster.sls +++ b/salt/elasticsearch/cluster.sls @@ -4,7 +4,7 @@ # Elastic License 2.0. {% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} +{% if sls in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} {% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %} diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index ab12b875e..77088d649 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -17,7 +17,7 @@ include: - elasticsearch.ssl - elasticsearch.config - elasticsearch.sostatus -{%- if GLOBALS.role != 'so-searchode' %} +{%- if GLOBALS.role != "so-searchode" %} - elasticsearch.cluster {%- endif%} @@ -102,11 +102,6 @@ so-elasticsearch: - cmd: auth_users_roles_inode - cmd: auth_users_inode -delete_so-elasticsearch_so-status.disabled: - file.uncomment: - - name: /opt/so/conf/so-status/so-status.conf - - regex: ^so-elasticsearch$ - wait_for_so-elasticsearch: http.wait_for_successful_query: - name: "https://localhost:9200/" @@ -117,10 +112,14 @@ wait_for_so-elasticsearch: - status: 200 - wait_for: 300 - request_interval: 15 - - backend: requests - require: - docker_container: so-elasticsearch +delete_so-elasticsearch_so-status.disabled: + file.uncomment: + - name: /opt/so/conf/so-status/so-status.conf + - regex: ^so-elasticsearch$ + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load index 840639a32..a0ebd66e8 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load @@ -103,11 +103,13 @@ load_component_templates() { local pattern="${ELASTICSEARCH_TEMPLATES_DIR}/component/$2" local append_mappings="${3:-"false"}" - # current state of nullglob shell option - shopt -q nullglob && nullglob_set=1 || nullglob_set=0 - - shopt -s nullglob echo -e "\nLoading $printed_name component templates...\n" + + if ! compgen -G "${pattern}/*.json" > /dev/null; then + echo "No $printed_name component templates found in ${pattern}, skipping." + return + fi + for component in "$pattern"/*.json; do tmpl_name=$(basename "${component%.json}") @@ -121,11 +123,6 @@ load_component_templates() { SO_LOAD_FAILURES_NAMES+=("$component") fi done - - # restore nullglob shell option if needed - if [[ $nullglob_set -eq 1 ]]; then - shopt -u nullglob - fi } check_elasticsearch_responsive() { @@ -136,7 +133,32 @@ check_elasticsearch_responsive() { fail "Elasticsearch is not responding. Please review Elasticsearch logs /opt/so/log/elasticsearch/securityonion.log for more details. Additionally, consider running so-elasticsearch-troubleshoot." } -if [[ "$FORCE" == "true" || ! -f "$SO_STATEFILE_SUCCESS" ]]; then +index_templates_exist() { + local templates_dir="$1" + + if [[ ! -d "$templates_dir" ]]; then + return 1 + fi + + compgen -G "${templates_dir}/*.json" > /dev/null +} + +should_load_addon_templates() { + if [[ "$IS_HEAVYNODE" == "true" ]]; then + return 1 + fi + + # Skip statefile checks when forcing template load + if [[ "$FORCE" != "true" ]]; then + if [[ ! -f "$SO_STATEFILE_SUCCESS" || -f "$ADDON_STATEFILE_SUCCESS" ]]; then + return 1 + fi + fi + + index_templates_exist "$ADDON_TEMPLATES_DIR" +} + +if [[ "$FORCE" == "true" || ! -f "$SO_STATEFILE_SUCCESS" ]] && index_templates_exist "$SO_TEMPLATES_DIR"; then check_elasticsearch_responsive if [[ "$IS_HEAVYNODE" == "false" ]]; then @@ -201,13 +223,14 @@ if [[ "$FORCE" == "true" || ! -f "$SO_STATEFILE_SUCCESS" ]]; then fail "Failed to load all Security Onion core templates successfully." fi fi -else - +elif ! index_templates_exist "$SO_TEMPLATES_DIR"; then + echo "No Security Onion core index templates found in ${SO_TEMPLATES_DIR}, skipping." +elif [[ -f "$SO_STATEFILE_SUCCESS" ]]; then echo "Security Onion core templates already loaded" fi # Start loading addon templates -if [[ (-d "$ADDON_TEMPLATES_DIR" && -f "$SO_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" && ! -f "$ADDON_STATEFILE_SUCCESS") || (-d "$ADDON_TEMPLATES_DIR" && "$IS_HEAVYNODE" == "false" && "$FORCE" == "true") ]]; then +if should_load_addon_templates; then check_elasticsearch_responsive diff --git a/salt/kibana/defaults.yaml b/salt/kibana/defaults.yaml index 580891973..ecf56756b 100644 --- a/salt/kibana/defaults.yaml +++ b/salt/kibana/defaults.yaml @@ -22,7 +22,7 @@ kibana: - default - file migrations: - discardCorruptObjects: "8.18.8" + discardCorruptObjects: "9.3.3" telemetry: enabled: False xpack: