From 152fdaa7bb4f826329244172ca8ec4e93cb32ed1 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 15 Apr 2025 11:40:43 -0400 Subject: [PATCH 01/19] Support Kratos user.name lookup --- .../grid-nodes_general/kratos-logs.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) rename salt/elasticfleet/files/{integrations => integrations-dynamic}/grid-nodes_general/kratos-logs.json (52%) diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json similarity index 52% rename from salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json rename to salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json index 6a67c9c1c..5ae81ba83 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json @@ -1,3 +1,4 @@ +{% set identities = salt['sqlite3.fetch']('/nsm/kratos/db/db.sqlite', 'SELECT id, json_extract(traits, "$.email") as email FROM identities;') %} { "package": { "name": "log", @@ -19,7 +20,7 @@ ], "data_stream.dataset": "kratos", "tags": ["so-kratos"], - "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: iam\n module: kratos", + "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- add_fields:\n target: event\n fields:\n category: iam\n module: kratos\n- if:\n has_fields:\n - identity_id\n then:{% for id, email in identities %}\n - if:\n equals:\n identity_id: \"{{ id }}\"\n then:\n - add_fields:\n target: ''\n fields:\n user.name: \"{{ email }}\"{% endfor %}", "custom": "pipeline: kratos" } } @@ -28,3 +29,4 @@ }, "force": true } + From 5fd7bf311d2310512b0d2fbac051c4a8b5a73e7e Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 15 Apr 2025 13:57:55 -0400 Subject: [PATCH 02/19] Add fallback --- .../grid-nodes_general/kratos-logs.json | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json index 5ae81ba83..f6b01cdff 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json @@ -1,4 +1,14 @@ -{% set identities = salt['sqlite3.fetch']('/nsm/kratos/db/db.sqlite', 'SELECT id, json_extract(traits, "$.email") as email FROM identities;') %} +{%- set identities = salt['sqlite3.fetch']('/nsm/kratos/db/db.sqlite', 'SELECT id, json_extract(traits, "$.email") as email FROM identities;') -%} +{%- set valid_identities = false -%} +{%- if identities -%} + {%- set valid_identities = true -%} + {%- for id, email in identities -%} + {%- if not id or not email -%} + {%- set valid_identities = false -%} + {%- break -%} + {%- endif -%} + {%- endfor -%} +{%- endif -%} { "package": { "name": "log", @@ -20,7 +30,11 @@ ], "data_stream.dataset": "kratos", "tags": ["so-kratos"], + {%- if valid_identities -%} "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- add_fields:\n target: event\n fields:\n category: iam\n module: kratos\n- if:\n has_fields:\n - identity_id\n then:{% for id, email in identities %}\n - if:\n equals:\n identity_id: \"{{ id }}\"\n then:\n - add_fields:\n target: ''\n fields:\n user.name: \"{{ email }}\"{% endfor %}", + {%- else -%} + "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- add_fields:\n target: event\n fields:\n category: iam\n module: kratos", + {%- endif -%} "custom": "pipeline: kratos" } } From 366e39950a5a8394e64d7c9a51c20446b0ae66dc Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 16 Apr 2025 15:55:16 -0400 Subject: [PATCH 03/19] subord annotations; ensure node reboots occur in background --- salt/manager/tools/sbin/so-minion | 2 +- salt/soc/soc_soc.yaml | 37 +++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index ebbfa8fff..98add9a96 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -126,7 +126,7 @@ function testMinion() { } function restartMinion() { - salt "$MINION_ID" system.reboot + salt "$MINION_ID" system.reboot --async result=$? exit $result diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index ac22aa2c1..48fef3dfb 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -138,6 +138,43 @@ soc: title: Require TOTP description: Require all users to enable Time-based One Time Passwords (MFA) upon login to SOC. global: True + subgrids: + title: Subordinate Grids + description: | + Optional list of *subgrids* that this grid has access to manage. This is also known as a 'Manager of Managers' configuration. The values entered must originate from the remote subordinate grid. The API Client must have be granted most permissions in order to perform required duties. + + *Requires a valid Security Onion license key with subgrid allocations.* + global: True + syntax: json + forcedType: "[]{}" + uiElements: + - field: id + label: Unique Subgrid ID + regex: "^(?!\s*all\s*$).*$" + regexFailureMessage: Subgrid ID cannot be named 'all' + required: true + - field: managerUrl + label: Subgrid Manager URL + required: true + - field: clientId + label: Subgrid API Client ID + required: true + regex: "^socl_[a-z0-9_]+$" + regexFailureMessage: Client ID must be a valid socl_* API Client ID + - field: clientSecret + label: Subgrid API Client Secret + required: true + - field: tlsSkipVerify + label: Skip Subgrid TLS Certification Validation + forcedType: bool + default: false + - field: caCertificate + label: Subgrid CA Certificate + multiline: True + - field: enabled + label: Subgrid Enabled + forcedType: bool + default: false modules: elastalertengine: aiRepoUrl: From 8f1e528f1c6ce8458cb39f4c1dc21c67d3b52512 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 17 Apr 2025 11:09:39 -0400 Subject: [PATCH 04/19] improve regex --- salt/soc/soc_soc.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 48fef3dfb..cac8c91d6 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -150,8 +150,8 @@ soc: uiElements: - field: id label: Unique Subgrid ID - regex: "^(?!\s*all\s*$).*$" - regexFailureMessage: Subgrid ID cannot be named 'all' + regex: "^_.*$" + regexFailureMessage: Subgrid ID cannot start with an underscore required: true - field: managerUrl label: Subgrid Manager URL From b607689993aea9eba6de80163251a778927a0044 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 17 Apr 2025 11:47:52 -0400 Subject: [PATCH 05/19] improve regex --- salt/soc/soc_soc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index cac8c91d6..dfaf7ab09 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -150,7 +150,7 @@ soc: uiElements: - field: id label: Unique Subgrid ID - regex: "^_.*$" + regex: "^((?!_)).+$" regexFailureMessage: Subgrid ID cannot start with an underscore required: true - field: managerUrl From d0375d3c7e75f1874beb1d5108cfd7526689547a Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 17 Apr 2025 11:51:21 -0400 Subject: [PATCH 06/19] fix typo --- salt/soc/soc_soc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index dfaf7ab09..7658d24dd 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -141,7 +141,7 @@ soc: subgrids: title: Subordinate Grids description: | - Optional list of *subgrids* that this grid has access to manage. This is also known as a 'Manager of Managers' configuration. The values entered must originate from the remote subordinate grid. The API Client must have be granted most permissions in order to perform required duties. + Optional list of *subgrids* that this grid has access to manage. This is also known as a 'Manager of Managers' configuration. The values entered must originate from the remote subordinate grid. The API Client must be granted most permissions in order to perform required duties. *Requires a valid Security Onion license key with subgrid allocations.* global: True From 3b447b343f6993b2d27fb6a6a6a6b9b01caf8dd9 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 17 Apr 2025 11:51:45 -0400 Subject: [PATCH 07/19] fix typo --- salt/soc/soc_soc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 7658d24dd..91ab6e3c1 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -141,7 +141,7 @@ soc: subgrids: title: Subordinate Grids description: | - Optional list of *subgrids* that this grid has access to manage. This is also known as a 'Manager of Managers' configuration. The values entered must originate from the remote subordinate grid. The API Client must be granted most permissions in order to perform required duties. + Optional list of *subgrids* that this grid has access to manage. This is also known as a 'Manager of Managers' configuration. The values entered must originate from the remote subordinate grid. The API Client must be granted most permissions in order to perform required functions. *Requires a valid Security Onion license key with subgrid allocations.* global: True From e3c8d22cac379a044c2d6b2d3f5a67f54b465afb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 18 Apr 2025 16:43:17 -0400 Subject: [PATCH 08/19] Update enabled.sls --- salt/elasticsearch/enabled.sls | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index af162d9e9..1d25fa668 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -204,12 +204,17 @@ so-elasticsearch-roles-load: - docker_container: so-elasticsearch - file: elasticsearch_sbin_jinja -{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %} +{% if grains.role in ['so-managersearch', 'so-heavynode', 'so-manager'] %} +{% set ap = "absent" %} +{% endif %} +{% if grains.role in ['so-eval', 'so-standalone'] %} {% if ELASTICSEARCHMERGED.index_clean %} {% set ap = "present" %} {% else %} {% set ap = "absent" %} {% endif %} +{% endif %} +{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %} so-elasticsearch-indices-delete: cron.{{ap}}: - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1 From c89adce3a1c53d06b3aff639117bc3a04194abd5 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 21 Apr 2025 10:48:18 -0500 Subject: [PATCH 09/19] default disable automatic upgrades for optional integration packages & policies --- salt/elasticfleet/defaults.yaml | 1 + salt/elasticfleet/enabled.sls | 4 ++ salt/elasticfleet/soc_elasticfleet.yaml | 4 ++ .../sbin/so-elastic-fleet-integration-upgrade | 62 ---------------- .../so-elastic-fleet-integration-upgrade | 72 +++++++++++++++++++ 5 files changed, 81 insertions(+), 62 deletions(-) delete mode 100644 salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-upgrade create mode 100644 salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index a0f509136..d6cdd7351 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -11,6 +11,7 @@ elasticfleet: defend_filters: enable_auto_configuration: False subscription_integrations: False + auto_upgrade_integrations: False logging: zeek: excluded: diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index 5a52f3a41..846203725 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -151,9 +151,13 @@ so-elastic-fleet-integration-upgrade: cmd.run: - name: /usr/sbin/so-elastic-fleet-integration-upgrade +{# CHECK THAT THIS STILL INSTALLS ALL INTEGRATIONS IN THE 2.4.130 SOUP #} +{# Subsequent runs should not modify the initially installed integration version.. until switch is flipped #} +{% if ELASTICFLEETMERGED.config.auto_upgrade_integrations %} so-elastic-fleet-addon-integrations: cmd.run: - name: /usr/sbin/so-elastic-fleet-optional-integrations-load +{% endif %} {% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %} so-elastic-defend-manage-filters-file-watch: diff --git a/salt/elasticfleet/soc_elasticfleet.yaml b/salt/elasticfleet/soc_elasticfleet.yaml index 7ca59401f..29439dfc0 100644 --- a/salt/elasticfleet/soc_elasticfleet.yaml +++ b/salt/elasticfleet/soc_elasticfleet.yaml @@ -45,6 +45,10 @@ elasticfleet: global: True forcedType: bool helpLink: elastic-fleet.html + auto_upgrade_integrations: + description: Enables or disables automatically upgrading Elastic Agent integrations. + global: True + helpLink: elastic-fleet.html server: custom_fqdn: description: Custom FQDN for Agents to connect to. One per line. diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-upgrade b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-upgrade deleted file mode 100644 index baad389eb..000000000 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-upgrade +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -. /usr/sbin/so-elastic-fleet-common - -curl_output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/) -if [ $? -ne 0 ]; then - echo "Error: Failed to connect to Kibana." - exit 1 -fi - -IFS=$'\n' -agent_policies=$(elastic_fleet_agent_policy_ids) -if [ $? -ne 0 ]; then - echo "Error: Failed to retrieve agent policies." - exit 1 -fi - -for AGENT_POLICY in $agent_policies; do - integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY") - for INTEGRATION in $integrations; do - if ! [[ "$INTEGRATION" == "elastic-defend-endpoints" ]] && ! [[ "$INTEGRATION" == "fleet_server-"* ]]; then - # Get package name so we know what package to look for when checking the current and latest available version - PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION") - - # Get currently installed version of package - PACKAGE_VERSION=$(elastic_fleet_integration_policy_package_version "$AGENT_POLICY" "$INTEGRATION") - - # Get latest available version of package - AVAILABLE_VERSION=$(elastic_fleet_package_latest_version_check "$PACKAGE_NAME") - - # Get integration ID - INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION") - - if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then - # Dry run of the upgrade - echo "Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..." - echo "Upgrading $INTEGRATION..." - echo "Starting dry run..." - DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID") - DRYRUN_ERRORS=$(echo "$DRYRUN_OUTPUT" | jq .[].hasErrors) - - # If no errors with dry run, proceed with actual upgrade - if [[ "$DRYRUN_ERRORS" == "false" ]]; then - echo "No errors detected. Proceeding with upgrade..." - elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID" - if [ $? -ne 0 ]; then - echo "Error: Upgrade failed for integration ID '$INTEGRATION_ID'." - exit 1 - fi - else - echo "Errors detected during dry run. Stopping upgrade..." - exit 1 - fi - fi - fi - done -done -echo diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade new file mode 100644 index 000000000..3c5bb06cb --- /dev/null +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade @@ -0,0 +1,72 @@ +#!/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. +{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} +{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %} +{%- set AUTO_UPGRADE_INTEGRATIONS = salt['pillar.get']('elasticfleet:config:auto_upgrade_integrations', default=ELASTICFLEETDEFAULTS.elasticfleet.config.auto_upgrade_integrations) %} + +. /usr/sbin/so-elastic-fleet-common + +curl_output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/) +if [ $? -ne 0 ]; then + echo "Error: Failed to connect to Kibana." + exit 1 +fi + +IFS=$'\n' +agent_policies=$(elastic_fleet_agent_policy_ids) +if [ $? -ne 0 ]; then + echo "Error: Failed to retrieve agent policies." + exit 1 +fi + +default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %}) + +for AGENT_POLICY in $agent_policies; do + integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY") + for INTEGRATION in $integrations; do + if ! [[ "$INTEGRATION" == "elastic-defend-endpoints" ]] && ! [[ "$INTEGRATION" == "fleet_server-"* ]]; then + # Get package name so we know what package to look for when checking the current and latest available version + PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION") + {%- if not AUTO_UPGRADE_INTEGRATIONS %} + if [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then + {%- endif %} + # Get currently installed version of package + PACKAGE_VERSION=$(elastic_fleet_integration_policy_package_version "$AGENT_POLICY" "$INTEGRATION") + + # Get latest available version of package + AVAILABLE_VERSION=$(elastic_fleet_package_latest_version_check "$PACKAGE_NAME") + + # Get integration ID + INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION") + + if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then + # Dry run of the upgrade + echo "Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..." + echo "Upgrading $INTEGRATION..." + echo "Starting dry run..." + DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID") + DRYRUN_ERRORS=$(echo "$DRYRUN_OUTPUT" | jq .[].hasErrors) + + # If no errors with dry run, proceed with actual upgrade + if [[ "$DRYRUN_ERRORS" == "false" ]]; then + echo "No errors detected. Proceeding with upgrade..." + elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID" + if [ $? -ne 0 ]; then + echo "Error: Upgrade failed for integration ID '$INTEGRATION_ID'." + exit 1 + fi + else + echo "Errors detected during dry run. Stopping upgrade..." + exit 1 + fi + fi + {%- if not AUTO_UPGRADE_INTEGRATIONS %} + fi + {%- endif %} + fi + done +done +echo From f5a8e917a49100e48dc16264c491c4add524c00c Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 21 Apr 2025 14:32:33 -0400 Subject: [PATCH 10/19] researching install failures --- salt/kratos/config.sls | 7 ++++++- salt/logstash/config.sls | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/salt/kratos/config.sls b/salt/kratos/config.sls index 0be43b460..0a54df9bf 100644 --- a/salt/kratos/config.sls +++ b/salt/kratos/config.sls @@ -19,7 +19,12 @@ kratos: - uid: 928 - gid: 928 - home: /opt/so/conf/kratos - + - retry: + attempts: 5 + until: True + interval: 60 + splay: 10 + kratosdir: file.directory: - name: /nsm/kratos diff --git a/salt/logstash/config.sls b/salt/logstash/config.sls index 8a59c83b7..37a724894 100644 --- a/salt/logstash/config.sls +++ b/salt/logstash/config.sls @@ -28,6 +28,11 @@ logstash: - uid: 931 - gid: 931 - home: /opt/so/conf/logstash + - retry: + attempts: 5 + until: True + interval: 60 + splay: 10 lslibdir: file.absent: From 4b7478654ffe7157f3c3c2f627ab7982f7d09eb6 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 21 Apr 2025 14:29:37 -0500 Subject: [PATCH 11/19] run optional integrations script so packages get installed. Hold updates unless auto_update_integrations is set --- salt/elasticfleet/enabled.sls | 4 ---- .../sbin_jinja/so-elastic-fleet-optional-integrations-load | 7 +++++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index 846203725..5a52f3a41 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -151,13 +151,9 @@ so-elastic-fleet-integration-upgrade: cmd.run: - name: /usr/sbin/so-elastic-fleet-integration-upgrade -{# CHECK THAT THIS STILL INSTALLS ALL INTEGRATIONS IN THE 2.4.130 SOUP #} -{# Subsequent runs should not modify the initially installed integration version.. until switch is flipped #} -{% if ELASTICFLEETMERGED.config.auto_upgrade_integrations %} so-elastic-fleet-addon-integrations: cmd.run: - name: /usr/sbin/so-elastic-fleet-optional-integrations-load -{% endif %} {% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %} so-elastic-defend-manage-filters-file-watch: diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load index f97ed577b..66fa235d1 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load @@ -4,6 +4,7 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. {% set SUB = salt['pillar.get']('elasticfleet:config:subscription_integrations', default=false) %} +{% set AUTO_UPGRADE_INTEGRATIONS = salt['pillar.get']('elasticfleet:config:auto_upgrade_integrations', default=false) %} . /usr/sbin/so-common . /usr/sbin/so-elastic-fleet-common @@ -74,6 +75,8 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST PENDING_UPDATE=true + {# only include upgrading optional integrations when auto_upgrade_integrations is true. DEFAULT integrations are upgraded using so-elastic-fleet-package-upgrade #} + {%- if AUTO_UPGRADE_INTEGRATIONS %} else results=$(compare_versions "$latest_version" "$installed_version") if [ $results == "greater" ]; then @@ -82,6 +85,7 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then PENDING_UPDATE=true fi + {%- endif %} fi fi {% else %} @@ -89,6 +93,8 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then echo "$package_name is not installed... Adding to next update." jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST PENDING_UPDATE=true + {# only include upgrading optional integrations when auto_upgrade_integrations is true. DEFAULT integrations are upgraded using so-elastic-fleet-package-upgrade #} + {%- if AUTO_UPGRADE_INTEGRATIONS %} else results=$(compare_versions "$latest_version" "$installed_version") if [ $results == "greater" ]; then @@ -96,6 +102,7 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST PENDING_UPDATE=true fi + {%- endif %} fi {% endif %} else From 166e4e0ebca3e3758cd71aeab5aa11ff5dd5a6e8 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 21 Apr 2025 15:51:36 -0500 Subject: [PATCH 12/19] make bool --- salt/elasticfleet/soc_elasticfleet.yaml | 1 + .../tools/sbin_jinja/so-elastic-fleet-integration-upgrade | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/elasticfleet/soc_elasticfleet.yaml b/salt/elasticfleet/soc_elasticfleet.yaml index 29439dfc0..450e044e6 100644 --- a/salt/elasticfleet/soc_elasticfleet.yaml +++ b/salt/elasticfleet/soc_elasticfleet.yaml @@ -48,6 +48,7 @@ elasticfleet: auto_upgrade_integrations: description: Enables or disables automatically upgrading Elastic Agent integrations. global: True + forcedType: bool helpLink: elastic-fleet.html server: custom_fqdn: diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade index 3c5bb06cb..8609395c1 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade @@ -5,7 +5,7 @@ # Elastic License 2.0. {%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} {%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %} -{%- set AUTO_UPGRADE_INTEGRATIONS = salt['pillar.get']('elasticfleet:config:auto_upgrade_integrations', default=ELASTICFLEETDEFAULTS.elasticfleet.config.auto_upgrade_integrations) %} +{%- set AUTO_UPGRADE_INTEGRATIONS = salt['pillar.get']('elasticfleet:config:auto_upgrade_integrations', default=false) %} . /usr/sbin/so-elastic-fleet-common From 4ec185a9c7a5c4ee55b5bce45fafad2c7313821c Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 21 Apr 2025 16:26:20 -0500 Subject: [PATCH 13/19] make logstash and kratos homedirs --- salt/kratos/config.sls | 8 ++++++++ salt/logstash/config.sls | 10 +++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/salt/kratos/config.sls b/salt/kratos/config.sls index 0a54df9bf..f26f380b4 100644 --- a/salt/kratos/config.sls +++ b/salt/kratos/config.sls @@ -13,6 +13,14 @@ kratosgroup: - name: kratos - gid: 928 +kratoshome: + file.directory: + - name: /opt/so/conf/kratos + - user: 928 + - gid: 928 + - mode: 700 + - makedirs: True + # Add Kratos user kratos: user.present: diff --git a/salt/logstash/config.sls b/salt/logstash/config.sls index 37a724894..f7daf3646 100644 --- a/salt/logstash/config.sls +++ b/salt/logstash/config.sls @@ -22,7 +22,15 @@ logstashgroup: - name: logstash - gid: 931 -# Add the logstash user for the jog4j settings +logstashhome: + file.directory: + - name: /opt/so/conf/logstash + - user: 931 + - gid: 931 + - mode: 700 + - makedirs: True + +# Add the logstash user for the log4j settings logstash: user.present: - uid: 931 From 30c4acb82875bdee7748c1e66d0508dbd3a2b32d Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 21 Apr 2025 16:38:16 -0500 Subject: [PATCH 14/19] group --- salt/kratos/config.sls | 7 +------ salt/logstash/config.sls | 7 +------ 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/salt/kratos/config.sls b/salt/kratos/config.sls index f26f380b4..b9f5142f1 100644 --- a/salt/kratos/config.sls +++ b/salt/kratos/config.sls @@ -17,7 +17,7 @@ kratoshome: file.directory: - name: /opt/so/conf/kratos - user: 928 - - gid: 928 + - group: 928 - mode: 700 - makedirs: True @@ -27,11 +27,6 @@ kratos: - uid: 928 - gid: 928 - home: /opt/so/conf/kratos - - retry: - attempts: 5 - until: True - interval: 60 - splay: 10 kratosdir: file.directory: diff --git a/salt/logstash/config.sls b/salt/logstash/config.sls index f7daf3646..5a1727e9b 100644 --- a/salt/logstash/config.sls +++ b/salt/logstash/config.sls @@ -26,7 +26,7 @@ logstashhome: file.directory: - name: /opt/so/conf/logstash - user: 931 - - gid: 931 + - group: 931 - mode: 700 - makedirs: True @@ -36,11 +36,6 @@ logstash: - uid: 931 - gid: 931 - home: /opt/so/conf/logstash - - retry: - attempts: 5 - until: True - interval: 60 - splay: 10 lslibdir: file.absent: From e17fea849a903a9cd0e06c0fc16de903ccfb574d Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 21 Apr 2025 20:32:42 -0500 Subject: [PATCH 15/19] continue loop after encountering error with first --- .../sbin_jinja/so-elastic-fleet-integration-upgrade | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade index 8609395c1..70148dd07 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade @@ -44,6 +44,7 @@ for AGENT_POLICY in $agent_policies; do if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then # Dry run of the upgrade + echo "" echo "Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..." echo "Upgrading $INTEGRATION..." echo "Starting dry run..." @@ -55,12 +56,12 @@ for AGENT_POLICY in $agent_policies; do echo "No errors detected. Proceeding with upgrade..." elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID" if [ $? -ne 0 ]; then - echo "Error: Upgrade failed for integration ID '$INTEGRATION_ID'." - exit 1 + echo "Error: Upgrade failed for $PACKAGE_NAME with integration ID '$INTEGRATION_ID'." + continue fi else - echo "Errors detected during dry run. Stopping upgrade..." - exit 1 + echo "Errors detected during dry run. Skipping $PACKAGE_NAME policy upgrade..." + continue fi fi {%- if not AUTO_UPGRADE_INTEGRATIONS %} From 8c4cf0ba08258b8f9bc1e9e1bc8865c053fe84fa Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 22 Apr 2025 07:29:12 -0500 Subject: [PATCH 16/19] keep hard failure --- .../tools/sbin_jinja/so-elastic-fleet-integration-upgrade | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade index 70148dd07..54540ba33 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade @@ -57,11 +57,11 @@ for AGENT_POLICY in $agent_policies; do elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID" if [ $? -ne 0 ]; then echo "Error: Upgrade failed for $PACKAGE_NAME with integration ID '$INTEGRATION_ID'." - continue + exit 1 fi else - echo "Errors detected during dry run. Skipping $PACKAGE_NAME policy upgrade..." - continue + echo "Errors detected during dry run for $PACKAGE_NAME policy upgrade..." + exit 1 fi fi {%- if not AUTO_UPGRADE_INTEGRATIONS %} From 559190aee3388434af190d64b9cd66ce40bdaf7e Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 22 Apr 2025 09:38:22 -0500 Subject: [PATCH 17/19] upgrade integrations if they aren't in an agent policy --- ...o-elastic-fleet-optional-integrations-load | 60 +++++++++++++++---- 1 file changed, 48 insertions(+), 12 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load index 66fa235d1..26d775e82 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load @@ -3,8 +3,10 @@ # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. +{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} {% set SUB = salt['pillar.get']('elasticfleet:config:subscription_integrations', default=false) %} {% set AUTO_UPGRADE_INTEGRATIONS = salt['pillar.get']('elasticfleet:config:auto_upgrade_integrations', default=false) %} +{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %} . /usr/sbin/so-common . /usr/sbin/so-elastic-fleet-common @@ -47,6 +49,28 @@ compare_versions() { fi } +IFS=$'\n' +agent_policies=$(elastic_fleet_agent_policy_ids) +if [ $? -ne 0 ]; then + echo "Error: Failed to retrieve agent policies." + exit 1 +fi + +default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %}) + +in_use_integrations=() + +for AGENT_POLICY in $agent_policies; do + integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY") + for INTEGRATION in $integrations; do + PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION") + # non-default integrations that are in-use in any policy + if ! [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then + in_use_integrations+=("$PACKAGE_NAME") + fi + done +done + if [[ -f $STATE_FILE_SUCCESS ]]; then if retry 3 1 "curl -s -K /opt/so/conf/elasticsearch/curl.config --output /dev/null --silent --head --fail localhost:5601/api/fleet/epm/packages"; then # Package_list contains all integrations beta / non-beta. @@ -75,17 +99,23 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST PENDING_UPDATE=true - {# only include upgrading optional integrations when auto_upgrade_integrations is true. DEFAULT integrations are upgraded using so-elastic-fleet-package-upgrade #} - {%- if AUTO_UPGRADE_INTEGRATIONS %} else results=$(compare_versions "$latest_version" "$installed_version") if [ $results == "greater" ]; then - echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update." - jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST + {#- When auto_upgrade_integrations is false, skip upgrading in_use_integrations #} + {%- if not AUTO_UPGRADE_INTEGRATIONS %} + if ! [[ " ${in_use_integrations[@]} " =~ " $package_name " ]]; then + {%- endif %} + echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update." + jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST - PENDING_UPDATE=true + PENDING_UPDATE=true + {%- if not AUTO_UPGRADE_INTEGRATIONS %} + else + echo "skipping available upgrade for in use integration - $package_name." + fi + {%- endif %} fi - {%- endif %} fi fi {% else %} @@ -93,16 +123,22 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then echo "$package_name is not installed... Adding to next update." jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST PENDING_UPDATE=true - {# only include upgrading optional integrations when auto_upgrade_integrations is true. DEFAULT integrations are upgraded using so-elastic-fleet-package-upgrade #} - {%- if AUTO_UPGRADE_INTEGRATIONS %} else results=$(compare_versions "$latest_version" "$installed_version") if [ $results == "greater" ]; then - echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update." - jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST - PENDING_UPDATE=true + {#- When auto_upgrade_integrations is false, skip upgrading in_use_integrations #} + {%- if not AUTO_UPGRADE_INTEGRATIONS %} + if ! [[ " ${in_use_integrations[@]} " =~ " $package_name " ]]; then + {%- endif %} + echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update." + jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST + PENDING_UPDATE=true + {%- if not AUTO_UPGRADE_INTEGRATIONS %} + else + echo "skipping available upgrade for in use integration - $package_name." + fi + {%- endif %} fi - {%- endif %} fi {% endif %} else From 77f88371b80433d242947350ec1148cc0adadca0 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 23 Apr 2025 08:30:37 -0400 Subject: [PATCH 18/19] manage default and local in separate states --- salt/elasticsearch/auth.sls | 2 +- salt/kafka/nodes.sls | 4 ++-- salt/kibana/secrets.sls | 2 +- salt/manager/elasticsearch.sls | 2 +- salt/manager/init.sls | 17 +++++++++++++++-- 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/salt/elasticsearch/auth.sls b/salt/elasticsearch/auth.sls index f3aefa6b9..a7de4ef8f 100644 --- a/salt/elasticsearch/auth.sls +++ b/salt/elasticsearch/auth.sls @@ -15,7 +15,7 @@ elastic_auth_pillar: file.managed: - name: /opt/so/saltstack/local/pillar/elasticsearch/auth.sls - - mode: 600 + - mode: 640 - reload_pillar: True - contents: | elasticsearch: diff --git a/salt/kafka/nodes.sls b/salt/kafka/nodes.sls index cae2a1d0f..90cc931bb 100644 --- a/salt/kafka/nodes.sls +++ b/salt/kafka/nodes.sls @@ -10,9 +10,9 @@ write_kafka_pillar_yaml: file.managed: - name: /opt/so/saltstack/local/pillar/kafka/nodes.sls - - mode: 644 + - mode: 640 - user: socore - source: salt://kafka/files/managed_node_pillar.jinja - template: jinja - context: - COMBINED_KAFKANODES: {{ COMBINED_KAFKANODES }} \ No newline at end of file + COMBINED_KAFKANODES: {{ COMBINED_KAFKANODES }} diff --git a/salt/kibana/secrets.sls b/salt/kibana/secrets.sls index f97aa4d59..048cea4d4 100644 --- a/salt/kibana/secrets.sls +++ b/salt/kibana/secrets.sls @@ -22,7 +22,7 @@ kibana_pillar_directory: kibana_secrets_pillar: file.managed: - name: /opt/so/saltstack/local/pillar/kibana/secrets.sls - - mode: 600 + - mode: 640 - reload_pillar: True - contents: | kibana: diff --git a/salt/manager/elasticsearch.sls b/salt/manager/elasticsearch.sls index df93217b8..ab9dbb287 100644 --- a/salt/manager/elasticsearch.sls +++ b/salt/manager/elasticsearch.sls @@ -3,5 +3,5 @@ elastic_curl_config_distributed: - name: /opt/so/saltstack/local/salt/elasticsearch/curl.config - source: salt://elasticsearch/files/curl.config.template - template: jinja - - mode: 600 + - mode: 640 - show_changes: False diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 5eadead92..4493047ba 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -127,15 +127,28 @@ so_fleetagent_status: - month: '*' - dayweek: '*' -socore_own_saltstack: +socore_own_saltstack_default: file.directory: - - name: /opt/so/saltstack + - name: /opt/so/saltstack/default - user: socore - group: socore - recurse: - user - group +socore_own_saltstack_local: + file.managed: + - name: /opt/so/saltstack/local + - user: socore + - group: socore + - dir_mode: 750 + - file_mode: 640 + - replace: False + - recurse: + - user + - group + - mode + rules_dir: file.directory: - name: /nsm/rules/yara From 19514a969b7a070b90bcba1d990f4fccd1ab15ea Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 23 Apr 2025 08:41:53 -0400 Subject: [PATCH 19/19] use file.directory --- salt/manager/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 4493047ba..07a1b8816 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -137,7 +137,7 @@ socore_own_saltstack_default: - group socore_own_saltstack_local: - file.managed: + file.directory: - name: /opt/so/saltstack/local - user: socore - group: socore