diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index a0f509136..d6cdd7351 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -11,6 +11,7 @@ elasticfleet: defend_filters: enable_auto_configuration: False subscription_integrations: False + auto_upgrade_integrations: False logging: zeek: excluded: diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json new file mode 100644 index 000000000..f6b01cdff --- /dev/null +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json @@ -0,0 +1,46 @@ +{%- set identities = salt['sqlite3.fetch']('/nsm/kratos/db/db.sqlite', 'SELECT id, json_extract(traits, "$.email") as email FROM identities;') -%} +{%- set valid_identities = false -%} +{%- if identities -%} + {%- set valid_identities = true -%} + {%- for id, email in identities -%} + {%- if not id or not email -%} + {%- set valid_identities = false -%} + {%- break -%} + {%- endif -%} + {%- endfor -%} +{%- endif -%} +{ + "package": { + "name": "log", + "version": "" + }, + "name": "kratos-logs", + "namespace": "so", + "description": "Kratos logs", + "policy_id": "so-grid-nodes_general", + "inputs": { + "logs-logfile": { + "enabled": true, + "streams": { + "log.logs": { + "enabled": true, + "vars": { + "paths": [ + "/opt/so/log/kratos/kratos.log" + ], + "data_stream.dataset": "kratos", + "tags": ["so-kratos"], + {%- if valid_identities -%} + "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- add_fields:\n target: event\n fields:\n category: iam\n module: kratos\n- if:\n has_fields:\n - identity_id\n then:{% for id, email in identities %}\n - if:\n equals:\n identity_id: \"{{ id }}\"\n then:\n - add_fields:\n target: ''\n fields:\n user.name: \"{{ email }}\"{% endfor %}", + {%- else -%} + "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- add_fields:\n target: event\n fields:\n category: iam\n module: kratos", + {%- endif -%} + "custom": "pipeline: kratos" + } + } + } + } + }, + "force": true +} + diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json deleted file mode 100644 index 6a67c9c1c..000000000 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "package": { - "name": "log", - "version": "" - }, - "name": "kratos-logs", - "namespace": "so", - "description": "Kratos logs", - "policy_id": "so-grid-nodes_general", - "inputs": { - "logs-logfile": { - "enabled": true, - "streams": { - "log.logs": { - "enabled": true, - "vars": { - "paths": [ - "/opt/so/log/kratos/kratos.log" - ], - "data_stream.dataset": "kratos", - "tags": ["so-kratos"], - "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: iam\n module: kratos", - "custom": "pipeline: kratos" - } - } - } - } - }, - "force": true -} diff --git a/salt/elasticfleet/soc_elasticfleet.yaml b/salt/elasticfleet/soc_elasticfleet.yaml index 7ca59401f..450e044e6 100644 --- a/salt/elasticfleet/soc_elasticfleet.yaml +++ b/salt/elasticfleet/soc_elasticfleet.yaml @@ -45,6 +45,11 @@ elasticfleet: global: True forcedType: bool helpLink: elastic-fleet.html + auto_upgrade_integrations: + description: Enables or disables automatically upgrading Elastic Agent integrations. + global: True + forcedType: bool + helpLink: elastic-fleet.html server: custom_fqdn: description: Custom FQDN for Agents to connect to. One per line. diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-upgrade b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-upgrade deleted file mode 100644 index baad389eb..000000000 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-upgrade +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -. /usr/sbin/so-elastic-fleet-common - -curl_output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/) -if [ $? -ne 0 ]; then - echo "Error: Failed to connect to Kibana." - exit 1 -fi - -IFS=$'\n' -agent_policies=$(elastic_fleet_agent_policy_ids) -if [ $? -ne 0 ]; then - echo "Error: Failed to retrieve agent policies." - exit 1 -fi - -for AGENT_POLICY in $agent_policies; do - integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY") - for INTEGRATION in $integrations; do - if ! [[ "$INTEGRATION" == "elastic-defend-endpoints" ]] && ! [[ "$INTEGRATION" == "fleet_server-"* ]]; then - # Get package name so we know what package to look for when checking the current and latest available version - PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION") - - # Get currently installed version of package - PACKAGE_VERSION=$(elastic_fleet_integration_policy_package_version "$AGENT_POLICY" "$INTEGRATION") - - # Get latest available version of package - AVAILABLE_VERSION=$(elastic_fleet_package_latest_version_check "$PACKAGE_NAME") - - # Get integration ID - INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION") - - if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then - # Dry run of the upgrade - echo "Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..." - echo "Upgrading $INTEGRATION..." - echo "Starting dry run..." - DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID") - DRYRUN_ERRORS=$(echo "$DRYRUN_OUTPUT" | jq .[].hasErrors) - - # If no errors with dry run, proceed with actual upgrade - if [[ "$DRYRUN_ERRORS" == "false" ]]; then - echo "No errors detected. Proceeding with upgrade..." - elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID" - if [ $? -ne 0 ]; then - echo "Error: Upgrade failed for integration ID '$INTEGRATION_ID'." - exit 1 - fi - else - echo "Errors detected during dry run. Stopping upgrade..." - exit 1 - fi - fi - fi - done -done -echo diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade new file mode 100644 index 000000000..54540ba33 --- /dev/null +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade @@ -0,0 +1,73 @@ +#!/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. +{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} +{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %} +{%- set AUTO_UPGRADE_INTEGRATIONS = salt['pillar.get']('elasticfleet:config:auto_upgrade_integrations', default=false) %} + +. /usr/sbin/so-elastic-fleet-common + +curl_output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/) +if [ $? -ne 0 ]; then + echo "Error: Failed to connect to Kibana." + exit 1 +fi + +IFS=$'\n' +agent_policies=$(elastic_fleet_agent_policy_ids) +if [ $? -ne 0 ]; then + echo "Error: Failed to retrieve agent policies." + exit 1 +fi + +default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %}) + +for AGENT_POLICY in $agent_policies; do + integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY") + for INTEGRATION in $integrations; do + if ! [[ "$INTEGRATION" == "elastic-defend-endpoints" ]] && ! [[ "$INTEGRATION" == "fleet_server-"* ]]; then + # Get package name so we know what package to look for when checking the current and latest available version + PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION") + {%- if not AUTO_UPGRADE_INTEGRATIONS %} + if [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then + {%- endif %} + # Get currently installed version of package + PACKAGE_VERSION=$(elastic_fleet_integration_policy_package_version "$AGENT_POLICY" "$INTEGRATION") + + # Get latest available version of package + AVAILABLE_VERSION=$(elastic_fleet_package_latest_version_check "$PACKAGE_NAME") + + # Get integration ID + INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION") + + if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then + # Dry run of the upgrade + echo "" + echo "Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..." + echo "Upgrading $INTEGRATION..." + echo "Starting dry run..." + DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID") + DRYRUN_ERRORS=$(echo "$DRYRUN_OUTPUT" | jq .[].hasErrors) + + # If no errors with dry run, proceed with actual upgrade + if [[ "$DRYRUN_ERRORS" == "false" ]]; then + echo "No errors detected. Proceeding with upgrade..." + elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID" + if [ $? -ne 0 ]; then + echo "Error: Upgrade failed for $PACKAGE_NAME with integration ID '$INTEGRATION_ID'." + exit 1 + fi + else + echo "Errors detected during dry run for $PACKAGE_NAME policy upgrade..." + exit 1 + fi + fi + {%- if not AUTO_UPGRADE_INTEGRATIONS %} + fi + {%- endif %} + fi + done +done +echo diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load index f97ed577b..26d775e82 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load @@ -3,7 +3,10 @@ # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. +{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} {% set SUB = salt['pillar.get']('elasticfleet:config:subscription_integrations', default=false) %} +{% set AUTO_UPGRADE_INTEGRATIONS = salt['pillar.get']('elasticfleet:config:auto_upgrade_integrations', default=false) %} +{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %} . /usr/sbin/so-common . /usr/sbin/so-elastic-fleet-common @@ -46,6 +49,28 @@ compare_versions() { fi } +IFS=$'\n' +agent_policies=$(elastic_fleet_agent_policy_ids) +if [ $? -ne 0 ]; then + echo "Error: Failed to retrieve agent policies." + exit 1 +fi + +default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %}) + +in_use_integrations=() + +for AGENT_POLICY in $agent_policies; do + integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY") + for INTEGRATION in $integrations; do + PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION") + # non-default integrations that are in-use in any policy + if ! [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then + in_use_integrations+=("$PACKAGE_NAME") + fi + done +done + if [[ -f $STATE_FILE_SUCCESS ]]; then if retry 3 1 "curl -s -K /opt/so/conf/elasticsearch/curl.config --output /dev/null --silent --head --fail localhost:5601/api/fleet/epm/packages"; then # Package_list contains all integrations beta / non-beta. @@ -77,10 +102,19 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then else results=$(compare_versions "$latest_version" "$installed_version") if [ $results == "greater" ]; then - echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update." - jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST + {#- When auto_upgrade_integrations is false, skip upgrading in_use_integrations #} + {%- if not AUTO_UPGRADE_INTEGRATIONS %} + if ! [[ " ${in_use_integrations[@]} " =~ " $package_name " ]]; then + {%- endif %} + echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update." + jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST - PENDING_UPDATE=true + PENDING_UPDATE=true + {%- if not AUTO_UPGRADE_INTEGRATIONS %} + else + echo "skipping available upgrade for in use integration - $package_name." + fi + {%- endif %} fi fi fi @@ -92,9 +126,18 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then else results=$(compare_versions "$latest_version" "$installed_version") if [ $results == "greater" ]; then - echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update." - jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST - PENDING_UPDATE=true + {#- When auto_upgrade_integrations is false, skip upgrading in_use_integrations #} + {%- if not AUTO_UPGRADE_INTEGRATIONS %} + if ! [[ " ${in_use_integrations[@]} " =~ " $package_name " ]]; then + {%- endif %} + echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update." + jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST + PENDING_UPDATE=true + {%- if not AUTO_UPGRADE_INTEGRATIONS %} + else + echo "skipping available upgrade for in use integration - $package_name." + fi + {%- endif %} fi fi {% endif %} diff --git a/salt/elasticsearch/auth.sls b/salt/elasticsearch/auth.sls index f3aefa6b9..a7de4ef8f 100644 --- a/salt/elasticsearch/auth.sls +++ b/salt/elasticsearch/auth.sls @@ -15,7 +15,7 @@ elastic_auth_pillar: file.managed: - name: /opt/so/saltstack/local/pillar/elasticsearch/auth.sls - - mode: 600 + - mode: 640 - reload_pillar: True - contents: | elasticsearch: diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index 4e1eecd0a..7b94dea00 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -204,12 +204,17 @@ so-elasticsearch-roles-load: - docker_container: so-elasticsearch - file: elasticsearch_sbin_jinja -{% if GLOBALS.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager', 'so-managerhype'] %} +{% if grains.role in ['so-managersearch', 'so-heavynode', 'so-manager', 'so-managerhype'] %} +{% set ap = "absent" %} +{% endif %} +{% if grains.role in ['so-eval', 'so-standalone'] %} {% if ELASTICSEARCHMERGED.index_clean %} {% set ap = "present" %} {% else %} {% set ap = "absent" %} {% endif %} +{% endif %} +{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %} so-elasticsearch-indices-delete: cron.{{ap}}: - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1 diff --git a/salt/kafka/nodes.sls b/salt/kafka/nodes.sls index cae2a1d0f..90cc931bb 100644 --- a/salt/kafka/nodes.sls +++ b/salt/kafka/nodes.sls @@ -10,9 +10,9 @@ write_kafka_pillar_yaml: file.managed: - name: /opt/so/saltstack/local/pillar/kafka/nodes.sls - - mode: 644 + - mode: 640 - user: socore - source: salt://kafka/files/managed_node_pillar.jinja - template: jinja - context: - COMBINED_KAFKANODES: {{ COMBINED_KAFKANODES }} \ No newline at end of file + COMBINED_KAFKANODES: {{ COMBINED_KAFKANODES }} diff --git a/salt/kibana/secrets.sls b/salt/kibana/secrets.sls index f97aa4d59..048cea4d4 100644 --- a/salt/kibana/secrets.sls +++ b/salt/kibana/secrets.sls @@ -22,7 +22,7 @@ kibana_pillar_directory: kibana_secrets_pillar: file.managed: - name: /opt/so/saltstack/local/pillar/kibana/secrets.sls - - mode: 600 + - mode: 640 - reload_pillar: True - contents: | kibana: diff --git a/salt/kratos/config.sls b/salt/kratos/config.sls index 0be43b460..b9f5142f1 100644 --- a/salt/kratos/config.sls +++ b/salt/kratos/config.sls @@ -13,13 +13,21 @@ kratosgroup: - name: kratos - gid: 928 +kratoshome: + file.directory: + - name: /opt/so/conf/kratos + - user: 928 + - group: 928 + - mode: 700 + - makedirs: True + # Add Kratos user kratos: user.present: - uid: 928 - gid: 928 - home: /opt/so/conf/kratos - + kratosdir: file.directory: - name: /nsm/kratos diff --git a/salt/logstash/config.sls b/salt/logstash/config.sls index 8a59c83b7..5a1727e9b 100644 --- a/salt/logstash/config.sls +++ b/salt/logstash/config.sls @@ -22,7 +22,15 @@ logstashgroup: - name: logstash - gid: 931 -# Add the logstash user for the jog4j settings +logstashhome: + file.directory: + - name: /opt/so/conf/logstash + - user: 931 + - group: 931 + - mode: 700 + - makedirs: True + +# Add the logstash user for the log4j settings logstash: user.present: - uid: 931 diff --git a/salt/manager/elasticsearch.sls b/salt/manager/elasticsearch.sls index df93217b8..ab9dbb287 100644 --- a/salt/manager/elasticsearch.sls +++ b/salt/manager/elasticsearch.sls @@ -3,5 +3,5 @@ elastic_curl_config_distributed: - name: /opt/so/saltstack/local/salt/elasticsearch/curl.config - source: salt://elasticsearch/files/curl.config.template - template: jinja - - mode: 600 + - mode: 640 - show_changes: False diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 5eadead92..07a1b8816 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -127,15 +127,28 @@ so_fleetagent_status: - month: '*' - dayweek: '*' -socore_own_saltstack: +socore_own_saltstack_default: file.directory: - - name: /opt/so/saltstack + - name: /opt/so/saltstack/default - user: socore - group: socore - recurse: - user - group +socore_own_saltstack_local: + file.directory: + - name: /opt/so/saltstack/local + - user: socore + - group: socore + - dir_mode: 750 + - file_mode: 640 + - replace: False + - recurse: + - user + - group + - mode + rules_dir: file.directory: - name: /nsm/rules/yara diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 211b036cf..55cd699ae 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -200,7 +200,7 @@ function testMinion() { function restartMinion() { log "INFO" "Restarting minion $MINION_ID" - salt "$MINION_ID" system.reboot + salt "$MINION_ID" system.reboot --async result=$? if [ $result -ne 0 ]; then diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index ac22aa2c1..91ab6e3c1 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -138,6 +138,43 @@ soc: title: Require TOTP description: Require all users to enable Time-based One Time Passwords (MFA) upon login to SOC. global: True + subgrids: + title: Subordinate Grids + description: | + Optional list of *subgrids* that this grid has access to manage. This is also known as a 'Manager of Managers' configuration. The values entered must originate from the remote subordinate grid. The API Client must be granted most permissions in order to perform required functions. + + *Requires a valid Security Onion license key with subgrid allocations.* + global: True + syntax: json + forcedType: "[]{}" + uiElements: + - field: id + label: Unique Subgrid ID + regex: "^((?!_)).+$" + regexFailureMessage: Subgrid ID cannot start with an underscore + required: true + - field: managerUrl + label: Subgrid Manager URL + required: true + - field: clientId + label: Subgrid API Client ID + required: true + regex: "^socl_[a-z0-9_]+$" + regexFailureMessage: Client ID must be a valid socl_* API Client ID + - field: clientSecret + label: Subgrid API Client Secret + required: true + - field: tlsSkipVerify + label: Skip Subgrid TLS Certification Validation + forcedType: bool + default: false + - field: caCertificate + label: Subgrid CA Certificate + multiline: True + - field: enabled + label: Subgrid Enabled + forcedType: bool + default: false modules: elastalertengine: aiRepoUrl: