From cd5483623b789613bf343d51a4585e8f9a249c80 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 18 Sep 2025 14:33:34 -0500 Subject: [PATCH] update import/eval fleet output config -- try to prevent corrupt dual 'default' output polices from having a successful installation --- .../tools/sbin_jinja/so-elastic-fleet-setup | 90 +++++++++++++++---- salt/manager/tools/sbin/soup | 28 ++++++ 2 files changed, 102 insertions(+), 16 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index 0510b6bfe..066edd2da 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -50,22 +50,54 @@ if [[ "$RETURN_CODE" != "0" ]]; then fi printf "\n### Create ES Token ###\n" -ESTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/service_tokens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq -r .value) +if ESTOKEN_RAW=$(fleet_api "service_tokens" -XPOST-H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then + ESTOKEN=$(echo "$ESTOKEN_RAW" | jq -r .value) +else + echo -e "\nFailed to create ES token..." + exit 1 +fi ### Create Outputs, Fleet Policy and Fleet URLs ### # Create the Manager Elasticsearch Output first and set it as the default output printf "\nAdd Manager Elasticsearch Output...\n" -ESCACRT=$(openssl x509 -in $INTCA) -JSON_STRING=$( jq -n \ - --arg ESCACRT "$ESCACRT" \ - '{"name":"so-manager_elasticsearch","id":"so-manager_elasticsearch","type":"elasticsearch","hosts":["https://{{ GLOBALS.manager_ip }}:9200","https://{{ GLOBALS.manager }}:9200"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate_authorities": [$ESCACRT]}}' ) -curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" +ESCACRT=$(openssl x509 -in "$INTCA" -outform DER | sha256sum | cut -d' ' -f1 | tr '[:lower:]' '[:upper:]') +JSON_STRING=$(jq -n \ + --arg ESCACRT "$ESCACRT" \ + '{"name":"so-manager_elasticsearch","id":"so-manager_elasticsearch","type":"elasticsearch","hosts":["https://{{ GLOBALS.manager_ip }}:9200","https://{{ GLOBALS.manager }}:9200"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ca_trusted_fingerprint": $ESCACRT}') + +if ! fleet_api "outputs" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then + echo -e "\nFailed to create so-elasticsearch_manager policy..." + exit 1 +fi printf "\n\n" +# At this point there should only be two policies. fleet-default-output & so-manager_elasticsearch +status "Verifying so-manager_elasticsearch policy is configured as the current default" + +# Grab the fleet-default-output policy instead of so-manager_elasticsearch, because a weird state can exist where both fleet-default-output & so-elasticsearch_manager can be set as the active default output for logs / metrics. Resulting in logs not ingesting on import/eval nodes +# Check that fleet-default-output isn't configured as a default for anything +if DEFAULTPOLICY=$(fleet_api "outputs/fleet-default-output"); then + fleet_default=$(echo "$DEFAULTPOLICY" | jq -er '.item.is_default') + fleet_default_monitoring=$(echo "$DEFAULTPOLICY" | jq -er '.item.is_default_monitoring') + if [[ ! $fleet_default ]] && [[ ! $fleet_default_monitoring ]]; then + echo -e "\nso-manager_elasticsearch is configured as the current default policy..." + else + echo -e "\nVerification of so-manager_elasticsearch policy failed... The default 'fleet-default-output' output is still active..." + exit 1 + fi +else + # fleet-output-policy is created automatically by fleet when started. Should always exist on any installation type + echo -e "\nDefault fleet-default-output policy doesn't exist...\n" + exit 1 +fi + # Create the Manager Fleet Server Host Agent Policy # This has to be done while the Elasticsearch Output is set to the default Output printf "Create Manager Fleet Server Policy...\n" -elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "false" "120" +if ! elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "false" "120"; then + echo -e "\n Failed to create Manager fleet server policy..." + exit 1 +fi # Modify the default integration policy to update the policy_id with the correct naming UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "FleetServer_{{ GLOBALS.hostname }}" --arg name "fleet_server-{{ GLOBALS.hostname }}" ' @@ -90,7 +122,10 @@ JSON_STRING=$( jq -n \ --arg LOGSTASHCA "$LOGSTASHCA" \ '{"name":"grid-logstash","is_default":true,"is_default_monitoring":true,"id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055", "{{ GLOBALS.manager }}:5055"],"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]},"proxy_id":null}' ) -curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" +if ! fleet_api "outputs" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then + echo -e "\nFailed to create logstash fleet output" + exit 1 +fi printf "\n\n" {%- endif %} @@ -108,7 +143,10 @@ else fi ## This array replaces whatever URLs are currently configured -curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/fleet_server_hosts" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" +if ! fleet_api "fleet_server_hosts" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then + echo -e "\nFailed to add manager fleet URL" + exit 1 +fi printf "\n\n" ### Create Policies & Associated Integration Configuration ### @@ -120,19 +158,19 @@ printf "\n\n" # Initial Endpoints Policy if ! elastic_fleet_policy_create "endpoints-initial" "Initial Endpoint Policy" "false" "1209600"; then - echo -e "Failed to create endpoints-initial policy..." + echo -e "\nFailed to create endpoints-initial policy..." exit 1 fi # Grid Nodes - General Policy if ! elastic_fleet_policy_create "so-grid-nodes_general" "SO Grid Nodes - General Purpose" "false" "1209600"; then - echo -e "Failed to create so-grid-nodes_general policy..." + echo -e "\nFailed to create so-grid-nodes_general policy..." exit 1 fi # Grid Nodes - Heavy Node Policy if ! elastic_fleet_policy_create "so-grid-nodes_heavy" "SO Grid Nodes - Heavy Node" "false" "1209600"; then - echo -e "Failed to create so-grid-nodes_heavy policy..." + echo -e "\nFailed to create so-grid-nodes_heavy policy..." exit 1 fi @@ -146,14 +184,34 @@ JSON_STRING=$( jq -n \ '{"name":$NAME,"host":$URL,"is_default":true}' ) -curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_download_sources" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" +if ! fleet_api "agent_download_sources" -XPOST-H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then + echo -e "\nFailed to update Elastic Agent artifact URL" + exit 1 +fi ### Finalization ### # Query for Enrollment Tokens for default policies -ENDPOINTSENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key') -GRIDNODESENROLLMENTOKENGENERAL=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_general")) | .api_key') -GRIDNODESENROLLMENTOKENHEAVY=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_heavy")) | .api_key') +if ENDPOINTSENROLLMENTOKEN_RAW=$(fleet_api "/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then + ENDPOINTSENROLLMENTOKEN=$(echo "$ENDPOINTSENROLLMENTOKEN_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key') +else + echo -e "\nFailed to query for Endpoints enrollment token" + exit 1 +fi + +if GRIDNODESENROLLMENTOKENGENERAL_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then + GRIDNODESENROLLMENTOKENGENERAL=$(echo "$GRIDNODESENROLLMENTOKENGENERAL_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_general")) | .api_key') +else + echo -e "\nFailed to query for Grid nodes - General enrollment token" + exit 1 +fi + +if GRIDNODESENROLLMENTOKENHEAVY_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then + GRIDNODESENROLLMENTOKENHEAVY=$(echo "$GRIDNODESENROLLMENTOKENHEAVY_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_heavy")) | .api_key') +else + echo -e "\nFailed to query for Grid nodes - Heavy enrollment token" + exit 1 +fi # Store needed data in minion pillar pillar_file=/opt/so/saltstack/local/pillar/minions/{{ GLOBALS.minion_id }}.sls diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 86595c162..46010da61 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -450,6 +450,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == 2.4.150 ]] && post_to_2.4.160 [[ "$POSTVERSION" == 2.4.160 ]] && post_to_2.4.170 [[ "$POSTVERSION" == 2.4.170 ]] && post_to_2.4.180 + [[ "$POSTVERSION" == 2.4.180 ]] && post_to_2.4.190 true } @@ -608,6 +609,15 @@ post_to_2.4.180() { POSTVERSION=2.4.180 } +post_to_2.4.190() { + # Only need to update import / eval nodes + if [[ "$MINIONID" =~ "_import" ]] || [[ ! "$MINIONID" =~ "_eval" ]]; then + update_import_fleet_output + fi + + POSTVERSION=2.4.190 +} + repo_sync() { echo "Sync the local repo." su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync." @@ -870,6 +880,11 @@ up_to_2.4.180() { INSTALLEDVERSION=2.4.180 } +up_to_2.4.190() { + echo "Nothing to do for 2.4.190" + INSTALLEDVERSION=2.4.190 +} + add_hydra_pillars() { mkdir -p /opt/so/saltstack/local/pillar/hydra touch /opt/so/saltstack/local/pillar/hydra/soc_hydra.sls @@ -1143,6 +1158,19 @@ update_elasticsearch_index_settings() { done } +update_import_fleet_output() { + if output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" --retry 3 --fail 2>/dev/null); then + # Update the current config of so-manager_elasticsearch output policy in place (leaving any customizations like having changed the preset value from 'balanced' to 'performance') + CAFINGERPRINT=$(openssl x509 -in /etc/pki/tls/certs/intca.crt -outform DER | sha256sum | cut -d' ' -f1 | tr '[:lower:]' '[:upper:]') + updated_policy=$(jq --args CAFINGERPRINT "$CAFINGERPRINT" '.item | (del(.id) | .ca_trusted_fingerprint = $CAFINGERPRINT)' <<< "$output") + if curl -sK /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" -XPUT -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$updated_policy" --retry 3 --fail 2>/dev/null; then + echo "Successfully updated so-manager_elasticsearch fleet output policy" + else + fail "Failed to update so-manager_elasticsearch fleet output policy" + fi + fi +} + update_salt_mine() { echo "Populating the mine with mine_functions for each host." set +e