diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index 166cb9719..60cf27deb 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -101,6 +101,17 @@ so-elastic-fleet: - file: trusttheca - x509: etc_elasticfleet_key - x509: etc_elasticfleet_crt + +wait_for_so-elastic-fleet: + http.wait_for_successful_query: + - name: "https://localhost:8220/api/status" + - ssl: True + - verify_ssl: False + - status: 200 + - wait_for: 300 + - request_interval: 15 + - require: + - docker_container: so-elastic-fleet {% endif %} delete_so-elastic-fleet_so-status.disabled: diff --git a/salt/elasticfleet/manager.sls b/salt/elasticfleet/manager.sls index 6cb672bef..c9fe91d4d 100644 --- a/salt/elasticfleet/manager.sls +++ b/salt/elasticfleet/manager.sls @@ -9,6 +9,7 @@ include: - elasticfleet.config + - kibana.enabled # If enabled, automatically update Fleet Logstash Outputs {% if ELASTICFLEETMERGED.config.server.enable_auto_configuration %} @@ -19,6 +20,8 @@ so-elastic-fleet-auto-configure-logstash-outputs: - retry: attempts: 4 interval: 30 + - require: + - http: wait_for_so-kibana {% endif %} # If enabled, automatically update Fleet Server URLs & ES Connection @@ -28,6 +31,8 @@ so-elastic-fleet-auto-configure-server-urls: - retry: attempts: 4 interval: 30 + - require: + - http: wait_for_so-kibana {% endif %} # Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs @@ -37,6 +42,8 @@ so-elastic-fleet-auto-configure-elasticsearch-urls: - retry: attempts: 4 interval: 30 + - require: + - http: wait_for_so-kibana so-elastic-fleet-auto-configure-artifact-urls: cmd.run: @@ -44,6 +51,8 @@ so-elastic-fleet-auto-configure-artifact-urls: - retry: attempts: 4 interval: 30 + - require: + - http: wait_for_so-kibana so-elastic-fleet-package-statefile: file.managed: @@ -55,7 +64,9 @@ so-elastic-fleet-package-upgrade: - name: /usr/sbin/so-elastic-fleet-package-upgrade - retry: attempts: 3 - interval: 10 + interval: 30 + - require: + - http: wait_for_so-kibana - onchanges: - file: /opt/so/state/elastic_fleet_packages.txt @@ -65,6 +76,8 @@ so-elastic-fleet-integrations: - retry: attempts: 3 interval: 10 + - require: + - http: wait_for_so-kibana so-elastic-agent-grid-upgrade: cmd.run: @@ -72,6 +85,8 @@ so-elastic-agent-grid-upgrade: - retry: attempts: 12 interval: 5 + - require: + - http: wait_for_so-kibana so-elastic-fleet-integration-upgrade: cmd.run: @@ -79,16 +94,22 @@ so-elastic-fleet-integration-upgrade: - retry: attempts: 3 interval: 10 + - require: + - http: wait_for_so-kibana {# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #} so-elastic-fleet-addon-integrations: cmd.run: - name: /usr/sbin/so-elastic-fleet-optional-integrations-load + - require: + - http: wait_for_so-kibana {% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %} so-elastic-defend-manage-filters-file-watch: cmd.run: - name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log + - require: + - http: wait_for_so-kibana - onchanges: - file: elasticdefendcustom - file: elasticdefenddisabled diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load index e548c7f86..81a3c74be 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load @@ -108,9 +108,12 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then done # Only create the state file if all policies were created/updated successfully - if [[ "$RETURN_CODE" != "1" ]]; then + if [[ $RETURN_CODE -eq 0 ]]; then touch /opt/so/state/eaintegrations.txt + else + exit 1 fi else - exit $RETURN_CODE + echo "Fleet integration policies already loaded." + exit 0 fi diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade index 18211a7c6..8ba250c00 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade @@ -8,18 +8,33 @@ . /usr/sbin/so-elastic-fleet-common +PKG_LOAD_FAILURES=0 +PKG_LOAD_FAILURES_NAMES=() + {%- for PACKAGE in SUPPORTED_PACKAGES %} echo "Upgrading {{ PACKAGE }} package..." if VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then - # exit 1 on failure to upgrade a default package, allow salt to handle retries - echo -e "\nERROR: Failed to upgrade $PACKAGE to version: $VERSION" - exit 1 + PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1)) + PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}") fi else - echo -e "\nERROR: Failed to get version information for integration $PACKAGE" + PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1)) + PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}") fi echo {%- endfor %} + +if [ $PKG_LOAD_FAILURES -gt 0 ]; then + echo "ERROR: Failed to upgrade $PKG_LOAD_FAILURES package(s):" + for PKG in "${PKG_LOAD_FAILURES_NAMES[@]}"; do + echo " - $PKG" + done + # exit 1 on failure to upgrade a default package, allow salt to handle retries + exit 1 +else + echo "Successfully upgraded all packages." +fi + echo /usr/sbin/so-elasticsearch-templates-load diff --git a/salt/kibana/enabled.sls b/salt/kibana/enabled.sls index 04f44e508..a2fb6cde9 100644 --- a/salt/kibana/enabled.sls +++ b/salt/kibana/enabled.sls @@ -6,6 +6,7 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls.split('.')[0] in allowed_states %} {% from 'docker/docker.map.jinja' import DOCKERMERGED %} +{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} {% from 'vars/globals.map.jinja' import GLOBALS %} include: @@ -60,6 +61,19 @@ so-kibana: - watch: - file: kibanaconfig +wait_for_so-kibana: + http.wait_for_successful_query: + - name: "http://localhost:5601/api/status" + - username: 'so_elastic' + - password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}' + - ssl: True + - verify_ssl: False + - status: 200 + - wait_for: 300 + - request_interval: 15 + - require: + - docker_container: so-kibana + delete_so-kibana_so-status.disabled: file.uncomment: - name: /opt/so/conf/so-status/so-status.conf