Merge pull request #15067 from Security-Onion-Solutions/vlb2

Vlb2

salt/common/tools/sbin/so-common

@@ -441,8 +441,7 @@ lookup_grain() {
 
 lookup_role() {
 	id=$(lookup_grain id)
-	pieces=($(echo $id | tr '_' ' '))
+	echo "${id##*_}"
-	echo ${pieces[1]}
 }
 
 is_feature_enabled() {

salt/elasticfleet/enabled.sls

@@ -135,12 +135,18 @@ so-elastic-fleet-package-statefile:
 so-elastic-fleet-package-upgrade:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-package-upgrade
+    - retry:
+        attempts: 3
+        interval: 10
     - onchanges:
       - file: /opt/so/state/elastic_fleet_packages.txt
 
 so-elastic-fleet-integrations:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-integration-policy-load
+    - retry:
+        attempts: 3
+        interval: 10
 
 so-elastic-agent-grid-upgrade:
   cmd.run:
@@ -152,7 +158,11 @@ so-elastic-agent-grid-upgrade:
 so-elastic-fleet-integration-upgrade:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-integration-upgrade
+    - retry:
+        attempts: 3
+        interval: 10
 
+{# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #}
 so-elastic-fleet-addon-integrations:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-optional-integrations-load

salt/elasticfleet/tools/sbin/so-elastic-fleet-common

@@ -23,6 +23,13 @@ fi
 # Define a banner to separate sections
 banner="========================================================================="
 
+fleet_api() {
+    local QUERYPATH=$1
+    shift
+
+    curl -sK /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/${QUERYPATH}" "$@" --retry 3 --fail 2>/dev/null
+}
+
 elastic_fleet_integration_check() {
 
     AGENT_POLICY=$1
@@ -39,7 +46,9 @@ elastic_fleet_integration_create() {
 
     JSON_STRING=$1
 
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+    if ! fleet_api "package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPOST -d "$JSON_STRING"; then
+        return 1
+    fi
 }
 
 
@@ -56,7 +65,10 @@ elastic_fleet_integration_remove() {
                     '{"packagePolicyIds":[$INTEGRATIONID]}'
                     )
 
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies/delete" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+    if ! fleet_api "package_policies/delete" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
+        echo "Error: Unable to delete '$NAME' from '$AGENT_POLICY'"
+        return 1
+    fi
 }
 
 elastic_fleet_integration_update() {
@@ -65,7 +77,9 @@ elastic_fleet_integration_update() {
 
     JSON_STRING=$2
 
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/package_policies/$UPDATE_ID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+    if ! fleet_api "package_policies/$UPDATE_ID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPUT -d "$JSON_STRING"; then
+        return 1
+    fi
 }
 
 elastic_fleet_integration_policy_upgrade() {
@@ -77,78 +91,83 @@ elastic_fleet_integration_policy_upgrade() {
                     '{"packagePolicyIds":[$INTEGRATIONID]}'
                     )
 
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies/upgrade" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+    if ! fleet_api "package_policies/upgrade" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
+        return 1
+    fi
 }
 
 
 elastic_fleet_package_version_check() {
     PACKAGE=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.version'
+
+    if output=$(fleet_api "epm/packages/$PACKAGE"); then
+        echo "$output" | jq -r '.item.version'
+    else
+        echo "Error: Failed to get current package version for '$PACKAGE'"
+        return 1
+    fi
 }
 
 elastic_fleet_package_latest_version_check() {
     PACKAGE=$1
-    if output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" --fail); then
+    if output=$(fleet_api "epm/packages/$PACKAGE"); then
         if version=$(jq -e -r '.item.latestVersion' <<< $output); then
             echo "$version"
         fi
     else
-        echo "Error: Failed to get latest version for $PACKAGE"
+        echo "Error: Failed to get latest version for '$PACKAGE'"
+        return 1
     fi
 }
 
 elastic_fleet_package_install() {
     PKG=$1
     VERSION=$2
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  -d '{"force":true}' "localhost:5601/api/fleet/epm/packages/$PKG/$VERSION"
+    if ! fleet_api "epm/packages/$PKG/$VERSION" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"force":true}'; then
+        return 1
+    fi
 }
 
 elastic_fleet_bulk_package_install() {
     BULK_PKG_LIST=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d@$1 "localhost:5601/api/fleet/epm/packages/_bulk"
+    if ! fleet_api "epm/packages/_bulk" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d@$BULK_PKG_LIST; then
-}
+        return 1
-
-elastic_fleet_package_is_installed() {
-    PACKAGE=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.status'
-}
-
-elastic_fleet_installed_packages() {
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' -H 'Content-Type: application/json' "localhost:5601/api/fleet/epm/packages/installed?perPage=500"
-}
-
-elastic_fleet_agent_policy_ids() {
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies" | jq -r .items[].id
-    if [ $? -ne 0 ]; then
-        echo "Error: Failed to retrieve agent policies."
-        exit 1
     fi
 }
 
-elastic_fleet_agent_policy_names() {
+elastic_fleet_installed_packages() {
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies" | jq -r .items[].name
+    if ! fleet_api "epm/packages/installed?perPage=500"; then
-    if [ $? -ne 0 ]; then
+        return 1
+    fi
+}
+
+elastic_fleet_agent_policy_ids() {
+    if output=$(fleet_api "agent_policies"); then
+        echo "$output" | jq -r .items[].id
+    else
         echo "Error: Failed to retrieve agent policies."
-        exit 1
+        return 1
     fi
 }
 
 elastic_fleet_integration_policy_names() {
     AGENT_POLICY=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r .item.package_policies[].name
+    if output=$(fleet_api "agent_policies/$AGENT_POLICY"); then
-    if [ $? -ne 0 ]; then
+        echo "$output" | jq -r .item.package_policies[].name
+    else
         echo "Error: Failed to retrieve integrations for '$AGENT_POLICY'."
-        exit 1
+        return 1
     fi
 }
 
 elastic_fleet_integration_policy_package_name() {
     AGENT_POLICY=$1
     INTEGRATION=$2
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.name'
+    if output=$(fleet_api "agent_policies/$AGENT_POLICY"); then
-    if [ $? -ne 0 ]; then
+        echo "$output" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.name'
+    else
         echo "Error: Failed to retrieve package name for '$INTEGRATION' in '$AGENT_POLICY'."
-        exit 1
+        return 1
     fi
 }
 
@@ -156,32 +175,32 @@ elastic_fleet_integration_policy_package_version() {
     AGENT_POLICY=$1
     INTEGRATION=$2
 
-    if output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" --fail); then
+    if output=$(fleet_api "agent_policies/$AGENT_POLICY"); then
-        if version=$(jq -e -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.version' <<< $output); then
+        if version=$(jq -e -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.version' <<< "$output"); then
             echo "$version"
         fi
     else
-        echo "Error: Failed to retrieve agent policy $AGENT_POLICY"
+        echo "Error: Failed to retrieve integration version for '$INTEGRATION' in policy '$AGENT_POLICY'"
-        exit 1
+        return 1
     fi
 }
 
 elastic_fleet_integration_id() {
     AGENT_POLICY=$1
     INTEGRATION=$2
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .id'
+    if output=$(fleet_api "agent_policies/$AGENT_POLICY"); then
-    if [ $? -ne 0 ]; then
+        echo "$output" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .id'
+    else
         echo "Error: Failed to retrieve integration ID for '$INTEGRATION' in '$AGENT_POLICY'."
-        exit 1
+        return 1
     fi
 }
 
 elastic_fleet_integration_policy_dryrun_upgrade() {
     INTEGRATION_ID=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -H "Content-Type: application/json" -H 'kbn-xsrf: true' -L -X POST "localhost:5601/api/fleet/package_policies/upgrade/dryrun" -d "{\"packagePolicyIds\":[\"$INTEGRATION_ID\"]}"
+    if ! fleet_api "package_policies/upgrade/dryrun" -H "Content-Type: application/json" -H 'kbn-xsrf: true' -XPOST -d "{\"packagePolicyIds\":[\"$INTEGRATION_ID\"]}"; then
-    if [ $? -ne 0 ]; then
         echo "Error: Failed to complete dry run for '$INTEGRATION_ID'."
-        exit 1
+        return 1
     fi
 }
 
@@ -190,25 +209,18 @@ elastic_fleet_policy_create() {
     NAME=$1
     DESC=$2
     FLEETSERVER=$3
-        TIMEOUT=$4
+    TIMEOUT=$4
 
     JSON_STRING=$( jq -n \
-                    --arg NAME "$NAME" \
+        --arg NAME "$NAME" \
-                    --arg DESC "$DESC" \
+        --arg DESC "$DESC" \
-                    --arg TIMEOUT $TIMEOUT \
+        --arg TIMEOUT $TIMEOUT \
-                    --arg FLEETSERVER "$FLEETSERVER" \
+        --arg FLEETSERVER "$FLEETSERVER" \
-                    '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":$TIMEOUT,"has_fleet_server":$FLEETSERVER}'
+            '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":$TIMEOUT,"has_fleet_server":$FLEETSERVER}'
-                    )
+        )
     # Create Fleet Policy
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+    if ! fleet_api "agent_policies" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
+        return 1
+    fi
 
 }
-
-elastic_fleet_policy_update() {
-
-    POLICYID=$1
-    JSON_STRING=$2
-
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/$POLICYID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
-}
-

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-defend

@@ -15,9 +15,15 @@ do
   elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"
   if [ -n "$INTEGRATION_ID" ]; then
       printf "\n\nIntegration $NAME exists - Upgrading integration policy\n"
-      elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"
+      if ! elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"; then
+        echo -e "\nFailed to upgrade integration policy for ${INTEGRATION##*/}"
+        exit 1
+      fi
   else
     printf "\n\nIntegration does not exist - Creating integration\n"
-    elastic_fleet_integration_create "@$INTEGRATION"
+    if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+        echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+        exit 1
+    fi
   fi
 done

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server

@@ -25,5 +25,9 @@ for POLICYNAME in $POLICY; do
     .name = $name' /opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json)
 
     # Now update the integration policy using the modified JSON
-    elastic_fleet_integration_update "$INTEGRATION_ID" "$UPDATED_INTEGRATION_POLICY"
+    if ! elastic_fleet_integration_update "$INTEGRATION_ID" "$UPDATED_INTEGRATION_POLICY"; then
+        # exit 1 on failure to update fleet integration policies, let salt handle retries
+        echo "Failed to update $POLICYNAME.."
+        exit 1
+    fi
 done

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -13,7 +13,7 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
   /usr/sbin/so-elastic-fleet-package-upgrade
 
   # Second, update Fleet Server policies
-  /sbin/so-elastic-fleet-integration-policy-elastic-fleet-server
+  /usr/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server
 
   # Third, configure Elastic Defend Integration seperately
   /usr/sbin/so-elastic-fleet-integration-policy-elastic-defend
@@ -25,10 +25,16 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
     elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"
     if [ -n "$INTEGRATION_ID" ]; then
       printf "\n\nIntegration $NAME exists - Updating integration\n"
-      elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"
+      if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then
+        echo -e "\nFailed to update integration for ${INTEGRATION##*/}"
+        exit 1
+      fi
     else
       printf "\n\nIntegration does not exist - Creating integration\n"
-      elastic_fleet_integration_create "@$INTEGRATION"
+      if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+        echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+        exit 1
+      fi
     fi
   done
 
@@ -39,10 +45,16 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
     elastic_fleet_integration_check "so-grid-nodes_general" "$INTEGRATION"
     if [ -n "$INTEGRATION_ID" ]; then
       printf "\n\nIntegration $NAME exists - Updating integration\n"
-      elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"
+      if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then
+        echo -e "\nFailed to update integration for ${INTEGRATION##*/}"
+        exit 1
+      fi
     else
       printf "\n\nIntegration does not exist - Creating integration\n"
-      elastic_fleet_integration_create "@$INTEGRATION"
+      if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+        echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+        exit 1
+      fi
     fi
   done
   if [[ "$RETURN_CODE" != "1" ]]; then
@@ -56,11 +68,17 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
     elastic_fleet_integration_check "so-grid-nodes_heavy" "$INTEGRATION"
     if [ -n "$INTEGRATION_ID" ]; then
       printf "\n\nIntegration $NAME exists - Updating integration\n"
-      elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"
+      if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then
+        echo -e "\nFailed to update integration for ${INTEGRATION##*/}"
+        exit 1
+      fi
     else
       printf "\n\nIntegration does not exist - Creating integration\n"
       if [ "$NAME" != "elasticsearch-logs" ]; then
-        elastic_fleet_integration_create "@$INTEGRATION"
+        if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+          echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+          exit 1
+        fi
       fi
     fi
   done
@@ -77,11 +95,17 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
       elastic_fleet_integration_check "$FLEET_POLICY" "$INTEGRATION"
       if [ -n "$INTEGRATION_ID" ]; then
         printf "\n\nIntegration $NAME exists - Updating integration\n"
-        elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"
+        if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then
+          echo -e "\nFailed to update integration for ${INTEGRATION##*/}"
+          exit 1
+        fi
       else
         printf "\n\nIntegration does not exist - Creating integration\n"
         if [ "$NAME" != "elasticsearch-logs" ]; then
-          elastic_fleet_integration_create "@$INTEGRATION"
+          if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+            echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+            exit 1
+          fi
         fi
       fi
     fi

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade

@@ -25,11 +25,16 @@ fi
 default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %})
 
 for AGENT_POLICY in $agent_policies; do
-    integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY")
+    if ! integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY"); then
+        # this script upgrades default integration packages, exit 1 and let salt handle retrying
+        exit 1
+    fi
     for INTEGRATION in $integrations; do
         if ! [[ "$INTEGRATION" == "elastic-defend-endpoints" ]] && ! [[ "$INTEGRATION" == "fleet_server-"* ]]; then
             # Get package name so we know what package to look for when checking the current and latest available version
-            PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION")
+            if ! PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION"); then
+                exit 1
+            fi
             {%- if not AUTO_UPGRADE_INTEGRATIONS %}
             if [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then
             {%- endif %}
@@ -48,7 +53,9 @@ for AGENT_POLICY in $agent_policies; do
                 fi
 
                 # Get integration ID
-                INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION")
+                if ! INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION"); then
+                    exit 1
+                fi
 
                 if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then
                     # Dry run of the upgrade
@@ -56,14 +63,15 @@ for AGENT_POLICY in $agent_policies; do
                     echo "Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..."
                     echo "Upgrading $INTEGRATION..."
                     echo "Starting dry run..."
-                    DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID")
+                    if ! DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID"); then
+                        exit 1
+                    fi
                     DRYRUN_ERRORS=$(echo "$DRYRUN_OUTPUT" | jq .[].hasErrors)
 
                     # If no errors with dry run, proceed with actual upgrade
                     if [[ "$DRYRUN_ERRORS" == "false" ]]; then
                         echo "No errors detected. Proceeding with upgrade..."
-                        elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"
+                        if ! elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"; then
-                        if [ $? -ne 0 ]; then
                             echo "Error: Upgrade failed for $PACKAGE_NAME with integration ID '$INTEGRATION_ID'."
                             exit 1
                         fi

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load

@@ -62,9 +62,17 @@ default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.l
 in_use_integrations=()
 
 for AGENT_POLICY in $agent_policies; do
-    integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY")
+
+    if ! integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY"); then
+        # skip the agent policy if we can't get required info, let salt retry. Integrations loaded by this script are non-default integrations.
+        echo "Skipping $AGENT_POLICY.. "
+        continue
+    fi
     for INTEGRATION in $integrations; do
-        PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION")
+        if ! PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION"); then
+            echo  "Not adding $INTEGRATION, couldn't get package name"
+            continue
+        fi
         # non-default integrations that are in-use in any policy
         if ! [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then
             in_use_integrations+=("$PACKAGE_NAME")
@@ -160,7 +168,11 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
 
             for file in "${pkg_filename}_"*.json; do
                 [ -e "$file" ] || continue
-                elastic_fleet_bulk_package_install $file >> $BULK_INSTALL_OUTPUT
+                if ! elastic_fleet_bulk_package_install $file >> $BULK_INSTALL_OUTPUT; then
+                    # integrations loaded my this script are non-essential and shouldn't cause exit, skip them for now next highstate run can retry
+                    echo "Failed to complete a chunk of bulk package installs -- $file "
+                    continue
+                fi
             done
             # cleanup any temp files for chunked package install
             rm -f ${pkg_filename}_*.json $BULK_INSTALL_PACKAGE_LIST
@@ -168,8 +180,9 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
             echo "Elastic integrations don't appear to need installation/updating..."
         fi
         # Write out file for generating index/component/ilm templates
-        latest_installed_package_list=$(elastic_fleet_installed_packages)
+        if latest_installed_package_list=$(elastic_fleet_installed_packages); then
-        echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS
+            echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS
+        fi
         if retry 3 1 "so-elasticsearch-query / --fail --output /dev/null"; then
             # Refresh installed component template list
             latest_component_templates_list=$(so-elasticsearch-query _component_template | jq '.component_templates[] | .name' | jq -s '.')

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load

@@ -10,8 +10,16 @@
 
 {%- for PACKAGE in SUPPORTED_PACKAGES %}
 echo "Setting up {{ PACKAGE }} package..."
-VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}")
+if VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}"); then
-elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"
+    if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then
+        # packages loaded by this script should never fail to install and REQUIRED before an installation of SO can be considered successful
+        echo -e "\nERROR: Failed to install default integration package -- $PACKAGE $VERSION"
+        exit 1
+    fi
+else
+    echo -e "\nERROR: Failed to get version information for integration $PACKAGE"
+    exit 1
+fi
 echo
 {%- endfor %}
 echo

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade

@@ -10,8 +10,15 @@
 
 {%- for PACKAGE in SUPPORTED_PACKAGES %}
 echo "Upgrading {{ PACKAGE }} package..."
-VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}")
+if VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
-elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"
+    if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then
+        # exit 1 on failure to upgrade a default package, allow salt to handle retries
+        echo -e "\nERROR: Failed to upgrade $PACKAGE to version: $VERSION"
+        exit 1
+    fi
+else
+    echo -e "\nERROR: Failed to get version information for integration $PACKAGE"
+fi
 echo
 {%- endfor %}
 echo

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup

@@ -23,18 +23,17 @@ if [[ "$RETURN_CODE" != "0" ]]; then
     exit 1
 fi
 
-ALIASES=".fleet-servers .fleet-policies-leader .fleet-policies .fleet-agents .fleet-artifacts .fleet-enrollment-api-keys .kibana_ingest"
+ALIASES=(.fleet-servers .fleet-policies-leader .fleet-policies .fleet-agents .fleet-artifacts .fleet-enrollment-api-keys .kibana_ingest)
-for ALIAS in ${ALIASES}
+for ALIAS in "${ALIASES[@]}"; do
-do
     # Get all concrete indices from alias
-    INDXS=$(curl -K /opt/so/conf/kibana/curl.config -s -k -L -H "Content-Type: application/json" "https://localhost:9200/_resolve/index/${ALIAS}"  | jq -r '.aliases[].indices[]')
+    if INDXS_RAW=$(curl -sK /opt/so/conf/kibana/curl.config -s -k -L -H "Content-Type: application/json" "https://localhost:9200/_resolve/index/${ALIAS}" --fail 2>/dev/null); then
-
+        INDXS=$(echo "$INDXS_RAW" | jq -r '.aliases[].indices[]')
-    # Delete all resolved indices
+        # Delete all resolved indices
-    for INDX in ${INDXS}
+        for INDX in ${INDXS}; do
-    do
             status "Deleting $INDX"
             curl -K /opt/so/conf/kibana/curl.config -s -k -L -H "Content-Type: application/json" "https://localhost:9200/${INDX}" -XDELETE
-    done
+        done
+    fi
 done
 
 # Restarting Kibana...
@@ -51,22 +50,54 @@ if [[ "$RETURN_CODE" != "0" ]]; then
 fi
 
 printf "\n### Create ES Token ###\n"
-ESTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/service_tokens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' |  jq -r .value)
+if ESTOKEN_RAW=$(fleet_api "service_tokens" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
+    ESTOKEN=$(echo "$ESTOKEN_RAW" | jq -r .value)
+else
+    echo -e "\nFailed to create ES token..."
+    exit 1
+fi
 
 ### Create Outputs, Fleet Policy and Fleet URLs ###
 # Create the Manager Elasticsearch Output first and set it as the default output
 printf "\nAdd Manager Elasticsearch Output...\n"
-ESCACRT=$(openssl x509 -in  $INTCA)
+ESCACRT=$(openssl x509 -in "$INTCA" -outform DER | sha256sum | cut -d' ' -f1 | tr '[:lower:]' '[:upper:]')
-JSON_STRING=$( jq -n \
+JSON_STRING=$(jq -n \
-                  --arg ESCACRT "$ESCACRT" \
+    --arg ESCACRT "$ESCACRT" \
-                  '{"name":"so-manager_elasticsearch","id":"so-manager_elasticsearch","type":"elasticsearch","hosts":["https://{{ GLOBALS.manager_ip }}:9200","https://{{ GLOBALS.manager }}:9200"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate_authorities": [$ESCACRT]}}' )
+        '{"name":"so-manager_elasticsearch","id":"so-manager_elasticsearch","type":"elasticsearch","hosts":["https://{{ GLOBALS.manager_ip }}:9200","https://{{ GLOBALS.manager }}:9200"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ca_trusted_fingerprint": $ESCACRT}')
-curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+
+if ! fleet_api "outputs" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
+    echo -e "\nFailed to create so-elasticsearch_manager policy..."
+    exit 1
+fi
 printf "\n\n"
 
+# At this point there should only be two policies. fleet-default-output & so-manager_elasticsearch
+status "Verifying so-manager_elasticsearch policy is configured as the current default"
+
+# Grab the fleet-default-output policy instead of so-manager_elasticsearch, because a weird state can exist where both fleet-default-output & so-elasticsearch_manager can be set as the active default output for logs / metrics. Resulting in logs not ingesting on import/eval nodes
+if DEFAULTPOLICY=$(fleet_api "outputs/fleet-default-output"); then
+    fleet_default=$(echo "$DEFAULTPOLICY" | jq -er '.item.is_default')
+    fleet_default_monitoring=$(echo "$DEFAULTPOLICY" | jq -er '.item.is_default_monitoring')
+#   Check that fleet-default-output isn't configured as a default for anything ( both variables return false )
+    if [[ ! $fleet_default ]] && [[ ! $fleet_default_monitoring ]]; then
+        echo -e "\nso-manager_elasticsearch is configured as the current default policy..."
+    else
+        echo -e "\nVerification of so-manager_elasticsearch policy failed... The default 'fleet-default-output' output is still active..."
+        exit 1
+    fi
+else
+    # fleet-output-policy is created automatically by fleet when started. Should always exist on any installation type
+    echo -e "\nDefault fleet-default-output policy doesn't exist...\n"
+    exit 1
+fi
+
 # Create the Manager Fleet Server Host Agent Policy
 # This has to be done while the Elasticsearch Output is set to the default Output
 printf "Create Manager Fleet Server Policy...\n"
-elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "false" "120"
+if ! elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "false" "120"; then
+    echo -e "\n Failed to create Manager fleet server policy..."
+    exit 1
+fi
 
 # Modify the default integration policy to update the policy_id with the correct naming
 UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "FleetServer_{{ GLOBALS.hostname }}" --arg name "fleet_server-{{ GLOBALS.hostname }}" '
@@ -74,7 +105,10 @@ UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "FleetServer_{{ GLOBALS.hostname
 .name = $name' /opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json)
 
 # Add the Fleet Server Integration to the new Fleet Policy
-elastic_fleet_integration_create "$UPDATED_INTEGRATION_POLICY"
+if ! elastic_fleet_integration_create "$UPDATED_INTEGRATION_POLICY"; then
+    echo -e "\nFailed to create Fleet server integration for Manager.."
+    exit 1
+fi
 
 # Now we can create the Logstash Output and set it to to be the default Output
 printf "\n\nCreate Logstash Output Config if node is not an Import or Eval install\n"
@@ -88,7 +122,10 @@ JSON_STRING=$( jq -n \
                   --arg LOGSTASHCA "$LOGSTASHCA" \
                   '{"name":"grid-logstash","is_default":true,"is_default_monitoring":true,"id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055", "{{ GLOBALS.manager }}:5055"],"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]},"proxy_id":null}'
                   )
-curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+if ! fleet_api "outputs" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
+    echo -e "\nFailed to create logstash fleet output"
+    exit 1
+fi
 printf "\n\n"
 {%- endif %}
 
@@ -106,7 +143,10 @@ else
 fi
 
 ## This array replaces whatever URLs are currently configured
-curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/fleet_server_hosts" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d  "$JSON_STRING"
+if ! fleet_api "fleet_server_hosts" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d  "$JSON_STRING"; then
+    echo -e "\nFailed to add manager fleet URL"
+    exit 1
+fi
 printf "\n\n"
 
 ### Create Policies & Associated Integration Configuration ###
@@ -117,13 +157,22 @@ printf "\n\n"
 /usr/sbin/so-elasticsearch-templates-load
 
 # Initial Endpoints Policy
-elastic_fleet_policy_create "endpoints-initial" "Initial Endpoint Policy" "false" "1209600"
+if ! elastic_fleet_policy_create "endpoints-initial" "Initial Endpoint Policy" "false" "1209600"; then
+    echo -e "\nFailed to create endpoints-initial policy..."
+    exit 1
+fi
 
 # Grid Nodes - General Policy
-elastic_fleet_policy_create "so-grid-nodes_general" "SO Grid Nodes - General Purpose" "false" "1209600"
+if ! elastic_fleet_policy_create "so-grid-nodes_general" "SO Grid Nodes - General Purpose" "false" "1209600"; then
+    echo -e "\nFailed to create so-grid-nodes_general policy..."
+    exit 1
+fi
 
 # Grid Nodes - Heavy Node Policy
-elastic_fleet_policy_create "so-grid-nodes_heavy" "SO Grid Nodes - Heavy Node" "false" "1209600"
+if ! elastic_fleet_policy_create "so-grid-nodes_heavy" "SO Grid Nodes - Heavy Node" "false" "1209600"; then
+    echo -e "\nFailed to create so-grid-nodes_heavy policy..."
+    exit 1
+fi
 
 # Load Integrations for default policies
 so-elastic-fleet-integration-policy-load
@@ -135,14 +184,34 @@ JSON_STRING=$( jq -n \
                 '{"name":$NAME,"host":$URL,"is_default":true}'
                 )
 
-curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_download_sources" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+if ! fleet_api "agent_download_sources" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
+    echo -e "\nFailed to update Elastic Agent artifact URL"
+    exit 1
+fi
 
 ### Finalization ###
 
 # Query for Enrollment Tokens for default policies
-ENDPOINTSENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
+if ENDPOINTSENROLLMENTOKEN_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
-GRIDNODESENROLLMENTOKENGENERAL=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_general")) | .api_key')
+    ENDPOINTSENROLLMENTOKEN=$(echo "$ENDPOINTSENROLLMENTOKEN_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
-GRIDNODESENROLLMENTOKENHEAVY=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_heavy")) | .api_key')
+else
+    echo -e "\nFailed to query for Endpoints enrollment token"
+    exit 1
+fi
+
+if GRIDNODESENROLLMENTOKENGENERAL_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
+    GRIDNODESENROLLMENTOKENGENERAL=$(echo "$GRIDNODESENROLLMENTOKENGENERAL_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_general")) | .api_key')
+else
+    echo -e "\nFailed to query for Grid nodes - General enrollment token"
+    exit 1
+fi
+
+if GRIDNODESENROLLMENTOKENHEAVY_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
+    GRIDNODESENROLLMENTOKENHEAVY=$(echo "$GRIDNODESENROLLMENTOKENHEAVY_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_heavy")) | .api_key')
+else
+    echo -e "\nFailed to query for Grid nodes - Heavy enrollment token"
+    exit 1
+fi
 
 # Store needed data in minion pillar
 pillar_file=/opt/so/saltstack/local/pillar/minions/{{ GLOBALS.minion_id }}.sls

salt/kratos/enabled.sls

@@ -54,6 +54,9 @@ so-kratos:
       - file: kratosconfig
       - file: kratoslogdir
       - file: kratosdir
+    - retry:
+        attempts: 10
+        interval: 10
 
 delete_so-kratos_so-status.disabled:
   file.uncomment:

salt/manager/tools/sbin/soup

@@ -450,6 +450,7 @@ postupgrade_changes() {
     [[ "$POSTVERSION"  == 2.4.150 ]] && post_to_2.4.160
     [[ "$POSTVERSION"  == 2.4.160 ]] && post_to_2.4.170
     [[ "$POSTVERSION"  == 2.4.170 ]] && post_to_2.4.180
+    [[ "$POSTVERSION"  == 2.4.180 ]] && post_to_2.4.190
     true
 }
 
@@ -608,6 +609,15 @@ post_to_2.4.180() {
   POSTVERSION=2.4.180
 }
 
+post_to_2.4.190() {
+    # Only need to update import / eval nodes
+    if [[ "$MINION_ROLE" == "import" ]] || [[ "$MINION_ROLE" == "eval" ]]; then
+        update_import_fleet_output
+    fi
+
+  POSTVERSION=2.4.190
+}
+
 repo_sync() {
   echo "Sync the local repo."
   su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync."
@@ -870,6 +880,11 @@ up_to_2.4.180() {
   INSTALLEDVERSION=2.4.180
 }
 
+up_to_2.4.190() {
+  echo "Nothing to do for 2.4.190"
+  INSTALLEDVERSION=2.4.190
+}
+
 add_hydra_pillars() {
   mkdir -p /opt/so/saltstack/local/pillar/hydra
   touch /opt/so/saltstack/local/pillar/hydra/soc_hydra.sls
@@ -1143,6 +1158,19 @@ update_elasticsearch_index_settings() {
   done
 }
 
+update_import_fleet_output() {
+    if output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" --retry 3 --fail 2>/dev/null); then
+        # Update the current config of so-manager_elasticsearch output policy in place (leaving any customizations like having changed the preset value from 'balanced' to 'performance')
+        CAFINGERPRINT=$(openssl x509 -in /etc/pki/tls/certs/intca.crt -outform DER | sha256sum | cut -d' ' -f1 | tr '[:lower:]' '[:upper:]')
+        updated_policy=$(jq --arg CAFINGERPRINT "$CAFINGERPRINT" '.item | (del(.id) | .ca_trusted_fingerprint = $CAFINGERPRINT)' <<< "$output")
+        if curl -sK /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" -XPUT -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$updated_policy" --retry 3 --fail 2>/dev/null; then
+            echo "Successfully updated so-manager_elasticsearch fleet output policy"
+        else
+            fail "Failed to update so-manager_elasticsearch fleet output policy"
+        fi
+    fi
+}
+
 update_salt_mine() {
     echo "Populating the mine with mine_functions for each host."
     set +e
@@ -1359,6 +1387,7 @@ main() {
   fi
 
   set_minionid
+  MINION_ROLE=$(lookup_role)
   echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
   echo ""
   if [[ $is_airgap -eq 0 ]]; then
@@ -1401,7 +1430,7 @@ main() {
   if [ "$is_hotfix" == "true" ]; then
     echo "Applying $HOTFIXVERSION hotfix"
     # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars
-    if [[ ! "$MINIONID" =~ "_import" ]]; then
+    if [[ ! "$MINION_ROLE" == "import" ]]; then
       backup_old_states_pillars
     fi
     copy_new_files
@@ -1464,7 +1493,7 @@ main() {
     fi
 
     # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars
-    if [[ ! "$MINIONID" =~ "_import" ]]; then
+    if [[ ! "$MINION_ROLE" == "import" ]]; then
       echo ""
       echo "Creating snapshots of default and local Salt states and pillars and saving to /nsm/backup/"
       backup_old_states_pillars

salt/salt/map.jinja

@@ -4,7 +4,10 @@
    Elastic License 2.0. #}
 
 {% set role = salt['grains.get']('role', '') %}
-{% if role in ['so-hypervisor','so-managerhype'] %}
+{# We are using usebr0 mostly for setup of the so-managerhype node and controlling when we use br0 vs the physical interface #}
+{% set usebr0 = salt['pillar.get']('usebr0', True) %}
+
+{% if role in ['so-hypervisor','so-managerhype'] and usebr0 %}
 {%   set interface = 'br0' %}
 {% else %}
 {%   set interface = pillar.host.mainint %}

salt/soc/defaults.yaml

@@ -2545,7 +2545,7 @@ soc:
               level: 'high'  # info | low | medium | high | critical
         assistant:
           enabled: false
-          investigationPrompt: Investigate Alert ID {socid}
+          investigationPrompt: Investigate Alert ID {socId}
           contextLimitSmall: 200000
           contextLimitLarge: 1000000
           thresholdColorRatioLow: 0.5

setup/so-functions

@@ -541,8 +541,15 @@ configure_minion() {
 		"log_file: /opt/so/log/salt/minion"\
 		"#startup_states: highstate" >> "$minion_config"
 
-	info "Running: salt-call state.apply salt.mine_functions --local --file-root=../salt/ -l info pillar='{"host": {"mainint": "$MNIC"}}'"
+	# At the time the so-managerhype node does not yet have the bridge configured.
-	salt-call state.apply salt.mine_functions --local --file-root=../salt/ -l info pillar="{'host': {'mainint': $MNIC}}"
+	# The so-hypervisor node doesn't either, but it doesn't cause issues here.
+	local usebr0=false
+	if [ "$minion_type" == 'hypervisor' ]; then
+    	usebr0=true
+	fi
+	local pillar_json="{\"host\": {\"mainint\": \"$MNIC\"}, \"usebr0\": $usebr0}"
+	info "Running: salt-call state.apply salt.mine_functions --local --file-root=../salt/ -l info pillar='$pillar_json'"
+	salt-call state.apply salt.mine_functions --local --file-root=../salt/ -l info pillar="$pillar_json"
 
 	{
 		logCmd "systemctl enable salt-minion";
@@ -1195,9 +1202,9 @@ hypervisor_local_states() {
         logCmd "salt-call state.apply libvirt.64962 --local --file-root=../salt/ -l info queue=True"
         info "Setting up bridge for $MNIC"
 		salt-call state.apply libvirt.bridge --local --file-root=../salt/ -l info pillar='{"host": {"mainint": "'$MNIC'"}}' queue=True
-		if [ $is_managerhype ]; then
+		#if [ $is_managerhype ]; then
-	    	logCmd "salt-call state.apply salt.minion queue=True"
+	    #	logCmd "salt-call state.apply salt.minion queue=True"
-		fi
+		#fi
     fi
 }
 

setup/so-setup

@@ -762,6 +762,7 @@ if ! [[ -f $install_opt_file ]]; then
 		fi
 		logCmd "salt-call state.apply common.packages"
 		logCmd "salt-call state.apply common"
+		hypervisor_local_states
 		# this will apply the salt.minion state first since salt.master includes salt.minion
 		logCmd "salt-call state.apply salt.master"
 		# wait here until we get a response from the salt-master since it may have just restarted
@@ -826,7 +827,6 @@ if ! [[ -f $install_opt_file ]]; then
 		checkin_at_boot
 		set_initial_firewall_access
 		logCmd "salt-call schedule.enable -linfo --local"
-		hypervisor_local_states
 		verify_setup
 	else
 		touch /root/accept_changes