update import/eval fleet output config -- try to prevent corrupt dual 'default' output polices from having a successful installation

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup

@@ -50,22 +50,54 @@ if [[ "$RETURN_CODE" != "0" ]]; then
 fi
 
 printf "\n### Create ES Token ###\n"
-ESTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/service_tokens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' |  jq -r .value)
+if ESTOKEN_RAW=$(fleet_api "service_tokens" -XPOST-H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
+    ESTOKEN=$(echo "$ESTOKEN_RAW" | jq -r .value)
+else
+    echo -e "\nFailed to create ES token..."
+    exit 1
+fi
 
 ### Create Outputs, Fleet Policy and Fleet URLs ###
 # Create the Manager Elasticsearch Output first and set it as the default output
 printf "\nAdd Manager Elasticsearch Output...\n"
-ESCACRT=$(openssl x509 -in  $INTCA)
+ESCACRT=$(openssl x509 -in "$INTCA" -outform DER | sha256sum | cut -d' ' -f1 | tr '[:lower:]' '[:upper:]')
-JSON_STRING=$( jq -n \
+JSON_STRING=$(jq -n \
-                  --arg ESCACRT "$ESCACRT" \
+    --arg ESCACRT "$ESCACRT" \
-                  '{"name":"so-manager_elasticsearch","id":"so-manager_elasticsearch","type":"elasticsearch","hosts":["https://{{ GLOBALS.manager_ip }}:9200","https://{{ GLOBALS.manager }}:9200"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate_authorities": [$ESCACRT]}}' )
+        '{"name":"so-manager_elasticsearch","id":"so-manager_elasticsearch","type":"elasticsearch","hosts":["https://{{ GLOBALS.manager_ip }}:9200","https://{{ GLOBALS.manager }}:9200"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ca_trusted_fingerprint": $ESCACRT}')
-curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+
+if ! fleet_api "outputs" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
+    echo -e "\nFailed to create so-elasticsearch_manager policy..."
+    exit 1
+fi
 printf "\n\n"
 
+# At this point there should only be two policies. fleet-default-output & so-manager_elasticsearch
+status "Verifying so-manager_elasticsearch policy is configured as the current default"
+
+# Grab the fleet-default-output policy instead of so-manager_elasticsearch, because a weird state can exist where both fleet-default-output & so-elasticsearch_manager can be set as the active default output for logs / metrics. Resulting in logs not ingesting on import/eval nodes
+#   Check that fleet-default-output isn't configured as a default for anything
+if DEFAULTPOLICY=$(fleet_api "outputs/fleet-default-output"); then
+    fleet_default=$(echo "$DEFAULTPOLICY" | jq -er '.item.is_default')
+    fleet_default_monitoring=$(echo "$DEFAULTPOLICY" | jq -er '.item.is_default_monitoring')
+    if [[ ! $fleet_default ]] && [[ ! $fleet_default_monitoring ]]; then
+        echo -e "\nso-manager_elasticsearch is configured as the current default policy..."
+    else
+        echo -e "\nVerification of so-manager_elasticsearch policy failed... The default 'fleet-default-output' output is still active..."
+        exit 1
+    fi
+else
+    # fleet-output-policy is created automatically by fleet when started. Should always exist on any installation type
+    echo -e "\nDefault fleet-default-output policy doesn't exist...\n"
+    exit 1
+fi
+
 # Create the Manager Fleet Server Host Agent Policy
 # This has to be done while the Elasticsearch Output is set to the default Output
 printf "Create Manager Fleet Server Policy...\n"
-elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "false" "120"
+if ! elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "false" "120"; then
+    echo -e "\n Failed to create Manager fleet server policy..."
+    exit 1
+fi
 
 # Modify the default integration policy to update the policy_id with the correct naming
 UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "FleetServer_{{ GLOBALS.hostname }}" --arg name "fleet_server-{{ GLOBALS.hostname }}" '
@@ -90,7 +122,10 @@ JSON_STRING=$( jq -n \
                   --arg LOGSTASHCA "$LOGSTASHCA" \
                   '{"name":"grid-logstash","is_default":true,"is_default_monitoring":true,"id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055", "{{ GLOBALS.manager }}:5055"],"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]},"proxy_id":null}'
                   )
-curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+if ! fleet_api "outputs" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
+    echo -e "\nFailed to create logstash fleet output"
+    exit 1
+fi
 printf "\n\n"
 {%- endif %}
 
@@ -108,7 +143,10 @@ else
 fi
 
 ## This array replaces whatever URLs are currently configured
-curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/fleet_server_hosts" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d  "$JSON_STRING"
+if ! fleet_api "fleet_server_hosts" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d  "$JSON_STRING"; then
+    echo -e "\nFailed to add manager fleet URL"
+    exit 1
+fi
 printf "\n\n"
 
 ### Create Policies & Associated Integration Configuration ###
@@ -120,19 +158,19 @@ printf "\n\n"
 
 # Initial Endpoints Policy
 if ! elastic_fleet_policy_create "endpoints-initial" "Initial Endpoint Policy" "false" "1209600"; then
-    echo -e "Failed to create endpoints-initial policy..."
+    echo -e "\nFailed to create endpoints-initial policy..."
     exit 1
 fi
 
 # Grid Nodes - General Policy
 if ! elastic_fleet_policy_create "so-grid-nodes_general" "SO Grid Nodes - General Purpose" "false" "1209600"; then
-    echo -e "Failed to create so-grid-nodes_general policy..."
+    echo -e "\nFailed to create so-grid-nodes_general policy..."
     exit 1
 fi
 
 # Grid Nodes - Heavy Node Policy
 if ! elastic_fleet_policy_create "so-grid-nodes_heavy" "SO Grid Nodes - Heavy Node" "false" "1209600"; then
-    echo -e "Failed to create so-grid-nodes_heavy policy..."
+    echo -e "\nFailed to create so-grid-nodes_heavy policy..."
     exit 1
 fi
 
@@ -146,14 +184,34 @@ JSON_STRING=$( jq -n \
                 '{"name":$NAME,"host":$URL,"is_default":true}'
                 )
 
-curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_download_sources" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+if ! fleet_api "agent_download_sources" -XPOST-H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
+    echo -e "\nFailed to update Elastic Agent artifact URL"
+    exit 1
+fi
 
 ### Finalization ###
 
 # Query for Enrollment Tokens for default policies
-ENDPOINTSENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
+if ENDPOINTSENROLLMENTOKEN_RAW=$(fleet_api "/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
-GRIDNODESENROLLMENTOKENGENERAL=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_general")) | .api_key')
+    ENDPOINTSENROLLMENTOKEN=$(echo "$ENDPOINTSENROLLMENTOKEN_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
-GRIDNODESENROLLMENTOKENHEAVY=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_heavy")) | .api_key')
+else
+    echo -e "\nFailed to query for Endpoints enrollment token"
+    exit 1
+fi
+
+if GRIDNODESENROLLMENTOKENGENERAL_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
+    GRIDNODESENROLLMENTOKENGENERAL=$(echo "$GRIDNODESENROLLMENTOKENGENERAL_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_general")) | .api_key')
+else
+    echo -e "\nFailed to query for Grid nodes - General enrollment token"
+    exit 1
+fi
+
+if GRIDNODESENROLLMENTOKENHEAVY_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
+    GRIDNODESENROLLMENTOKENHEAVY=$(echo "$GRIDNODESENROLLMENTOKENHEAVY_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_heavy")) | .api_key')
+else
+    echo -e "\nFailed to query for Grid nodes - Heavy enrollment token"
+    exit 1
+fi
 
 # Store needed data in minion pillar
 pillar_file=/opt/so/saltstack/local/pillar/minions/{{ GLOBALS.minion_id }}.sls

salt/manager/tools/sbin/soup

@@ -450,6 +450,7 @@ postupgrade_changes() {
     [[ "$POSTVERSION"  == 2.4.150 ]] && post_to_2.4.160
     [[ "$POSTVERSION"  == 2.4.160 ]] && post_to_2.4.170
     [[ "$POSTVERSION"  == 2.4.170 ]] && post_to_2.4.180
+    [[ "$POSTVERSION"  == 2.4.180 ]] && post_to_2.4.190
     true
 }
 
@@ -608,6 +609,15 @@ post_to_2.4.180() {
   POSTVERSION=2.4.180
 }
 
+post_to_2.4.190() {
+    # Only need to update import / eval nodes
+    if [[ "$MINIONID" =~ "_import" ]] || [[ ! "$MINIONID" =~ "_eval" ]]; then
+        update_import_fleet_output
+    fi
+
+  POSTVERSION=2.4.190
+}
+
 repo_sync() {
   echo "Sync the local repo."
   su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync."
@@ -870,6 +880,11 @@ up_to_2.4.180() {
   INSTALLEDVERSION=2.4.180
 }
 
+up_to_2.4.190() {
+  echo "Nothing to do for 2.4.190"
+  INSTALLEDVERSION=2.4.190
+}
+
 add_hydra_pillars() {
   mkdir -p /opt/so/saltstack/local/pillar/hydra
   touch /opt/so/saltstack/local/pillar/hydra/soc_hydra.sls
@@ -1143,6 +1158,19 @@ update_elasticsearch_index_settings() {
   done
 }
 
+update_import_fleet_output() {
+    if output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" --retry 3 --fail 2>/dev/null); then
+        # Update the current config of so-manager_elasticsearch output policy in place (leaving any customizations like having changed the preset value from 'balanced' to 'performance')
+        CAFINGERPRINT=$(openssl x509 -in /etc/pki/tls/certs/intca.crt -outform DER | sha256sum | cut -d' ' -f1 | tr '[:lower:]' '[:upper:]')
+        updated_policy=$(jq --args CAFINGERPRINT "$CAFINGERPRINT" '.item | (del(.id) | .ca_trusted_fingerprint = $CAFINGERPRINT)' <<< "$output")
+        if curl -sK /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" -XPUT -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$updated_policy" --retry 3 --fail 2>/dev/null; then
+            echo "Successfully updated so-manager_elasticsearch fleet output policy"
+        else
+            fail "Failed to update so-manager_elasticsearch fleet output policy"
+        fi
+    fi
+}
+
 update_salt_mine() {
     echo "Populating the mine with mine_functions for each host."
     set +e