diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 203b54cd0..365852e63 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -441,8 +441,7 @@ lookup_grain() { lookup_role() { id=$(lookup_grain id) - pieces=($(echo $id | tr '_' ' ')) - echo ${pieces[1]} + echo "${id##*_}" } is_feature_enabled() { diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index 0ca54ccb8..cef47168f 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -135,12 +135,18 @@ so-elastic-fleet-package-statefile: so-elastic-fleet-package-upgrade: cmd.run: - name: /usr/sbin/so-elastic-fleet-package-upgrade + - retry: + attempts: 3 + interval: 10 - onchanges: - file: /opt/so/state/elastic_fleet_packages.txt so-elastic-fleet-integrations: cmd.run: - name: /usr/sbin/so-elastic-fleet-integration-policy-load + - retry: + attempts: 3 + interval: 10 so-elastic-agent-grid-upgrade: cmd.run: @@ -152,7 +158,11 @@ so-elastic-agent-grid-upgrade: so-elastic-fleet-integration-upgrade: cmd.run: - name: /usr/sbin/so-elastic-fleet-integration-upgrade + - retry: + attempts: 3 + interval: 10 +{# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #} so-elastic-fleet-addon-integrations: cmd.run: - name: /usr/sbin/so-elastic-fleet-optional-integrations-load diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common index 9780c8b12..4ca5030aa 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common @@ -23,6 +23,13 @@ fi # Define a banner to separate sections banner="=========================================================================" +fleet_api() { + local QUERYPATH=$1 + shift + + curl -sK /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/${QUERYPATH}" "$@" --retry 3 --fail 2>/dev/null +} + elastic_fleet_integration_check() { AGENT_POLICY=$1 @@ -39,7 +46,9 @@ elastic_fleet_integration_create() { JSON_STRING=$1 - curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" + if ! fleet_api "package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPOST -d "$JSON_STRING"; then + return 1 + fi } @@ -56,7 +65,10 @@ elastic_fleet_integration_remove() { '{"packagePolicyIds":[$INTEGRATIONID]}' ) - curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies/delete" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" + if ! fleet_api "package_policies/delete" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then + echo "Error: Unable to delete '$NAME' from '$AGENT_POLICY'" + return 1 + fi } elastic_fleet_integration_update() { @@ -65,7 +77,9 @@ elastic_fleet_integration_update() { JSON_STRING=$2 - curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/package_policies/$UPDATE_ID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" + if ! fleet_api "package_policies/$UPDATE_ID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPUT -d "$JSON_STRING"; then + return 1 + fi } elastic_fleet_integration_policy_upgrade() { @@ -77,78 +91,83 @@ elastic_fleet_integration_policy_upgrade() { '{"packagePolicyIds":[$INTEGRATIONID]}' ) - curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies/upgrade" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" + if ! fleet_api "package_policies/upgrade" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then + return 1 + fi } elastic_fleet_package_version_check() { PACKAGE=$1 - curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.version' + + if output=$(fleet_api "epm/packages/$PACKAGE"); then + echo "$output" | jq -r '.item.version' + else + echo "Error: Failed to get current package version for '$PACKAGE'" + return 1 + fi } elastic_fleet_package_latest_version_check() { PACKAGE=$1 - if output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" --fail); then + if output=$(fleet_api "epm/packages/$PACKAGE"); then if version=$(jq -e -r '.item.latestVersion' <<< $output); then echo "$version" fi else - echo "Error: Failed to get latest version for $PACKAGE" + echo "Error: Failed to get latest version for '$PACKAGE'" + return 1 fi } elastic_fleet_package_install() { PKG=$1 VERSION=$2 - curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"force":true}' "localhost:5601/api/fleet/epm/packages/$PKG/$VERSION" + if ! fleet_api "epm/packages/$PKG/$VERSION" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"force":true}'; then + return 1 + fi } elastic_fleet_bulk_package_install() { BULK_PKG_LIST=$1 - curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d@$1 "localhost:5601/api/fleet/epm/packages/_bulk" -} - -elastic_fleet_package_is_installed() { - PACKAGE=$1 - curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.status' -} - -elastic_fleet_installed_packages() { - curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' -H 'Content-Type: application/json' "localhost:5601/api/fleet/epm/packages/installed?perPage=500" -} - -elastic_fleet_agent_policy_ids() { - curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies" | jq -r .items[].id - if [ $? -ne 0 ]; then - echo "Error: Failed to retrieve agent policies." - exit 1 + if ! fleet_api "epm/packages/_bulk" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d@$BULK_PKG_LIST; then + return 1 fi } -elastic_fleet_agent_policy_names() { - curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies" | jq -r .items[].name - if [ $? -ne 0 ]; then +elastic_fleet_installed_packages() { + if ! fleet_api "epm/packages/installed?perPage=500"; then + return 1 + fi +} + +elastic_fleet_agent_policy_ids() { + if output=$(fleet_api "agent_policies"); then + echo "$output" | jq -r .items[].id + else echo "Error: Failed to retrieve agent policies." - exit 1 + return 1 fi } elastic_fleet_integration_policy_names() { AGENT_POLICY=$1 - curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r .item.package_policies[].name - if [ $? -ne 0 ]; then + if output=$(fleet_api "agent_policies/$AGENT_POLICY"); then + echo "$output" | jq -r .item.package_policies[].name + else echo "Error: Failed to retrieve integrations for '$AGENT_POLICY'." - exit 1 + return 1 fi } elastic_fleet_integration_policy_package_name() { AGENT_POLICY=$1 INTEGRATION=$2 - curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.name' - if [ $? -ne 0 ]; then + if output=$(fleet_api "agent_policies/$AGENT_POLICY"); then + echo "$output" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.name' + else echo "Error: Failed to retrieve package name for '$INTEGRATION' in '$AGENT_POLICY'." - exit 1 + return 1 fi } @@ -156,32 +175,32 @@ elastic_fleet_integration_policy_package_version() { AGENT_POLICY=$1 INTEGRATION=$2 - if output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" --fail); then - if version=$(jq -e -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.version' <<< $output); then + if output=$(fleet_api "agent_policies/$AGENT_POLICY"); then + if version=$(jq -e -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.version' <<< "$output"); then echo "$version" fi else - echo "Error: Failed to retrieve agent policy $AGENT_POLICY" - exit 1 + echo "Error: Failed to retrieve integration version for '$INTEGRATION' in policy '$AGENT_POLICY'" + return 1 fi } elastic_fleet_integration_id() { AGENT_POLICY=$1 INTEGRATION=$2 - curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .id' - if [ $? -ne 0 ]; then + if output=$(fleet_api "agent_policies/$AGENT_POLICY"); then + echo "$output" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .id' + else echo "Error: Failed to retrieve integration ID for '$INTEGRATION' in '$AGENT_POLICY'." - exit 1 + return 1 fi } elastic_fleet_integration_policy_dryrun_upgrade() { INTEGRATION_ID=$1 - curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -H "Content-Type: application/json" -H 'kbn-xsrf: true' -L -X POST "localhost:5601/api/fleet/package_policies/upgrade/dryrun" -d "{\"packagePolicyIds\":[\"$INTEGRATION_ID\"]}" - if [ $? -ne 0 ]; then + if ! fleet_api "package_policies/upgrade/dryrun" -H "Content-Type: application/json" -H 'kbn-xsrf: true' -XPOST -d "{\"packagePolicyIds\":[\"$INTEGRATION_ID\"]}"; then echo "Error: Failed to complete dry run for '$INTEGRATION_ID'." - exit 1 + return 1 fi } @@ -190,25 +209,18 @@ elastic_fleet_policy_create() { NAME=$1 DESC=$2 FLEETSERVER=$3 - TIMEOUT=$4 + TIMEOUT=$4 JSON_STRING=$( jq -n \ - --arg NAME "$NAME" \ - --arg DESC "$DESC" \ - --arg TIMEOUT $TIMEOUT \ - --arg FLEETSERVER "$FLEETSERVER" \ - '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":$TIMEOUT,"has_fleet_server":$FLEETSERVER}' - ) + --arg NAME "$NAME" \ + --arg DESC "$DESC" \ + --arg TIMEOUT $TIMEOUT \ + --arg FLEETSERVER "$FLEETSERVER" \ + '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":$TIMEOUT,"has_fleet_server":$FLEETSERVER}' + ) # Create Fleet Policy - curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" + if ! fleet_api "agent_policies" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then + return 1 + fi } - -elastic_fleet_policy_update() { - - POLICYID=$1 - JSON_STRING=$2 - - curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/$POLICYID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" -} - diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-defend b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-defend index 636942490..9769f2f79 100755 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-defend +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-defend @@ -15,9 +15,15 @@ do elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION" if [ -n "$INTEGRATION_ID" ]; then printf "\n\nIntegration $NAME exists - Upgrading integration policy\n" - elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID" + if ! elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"; then + echo -e "\nFailed to upgrade integration policy for ${INTEGRATION##*/}" + exit 1 + fi else printf "\n\nIntegration does not exist - Creating integration\n" - elastic_fleet_integration_create "@$INTEGRATION" + if ! elastic_fleet_integration_create "@$INTEGRATION"; then + echo -e "\nFailed to create integration for ${INTEGRATION##*/}" + exit 1 + fi fi done diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server index 8f7c8b8b4..caa684829 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server @@ -25,5 +25,9 @@ for POLICYNAME in $POLICY; do .name = $name' /opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json) # Now update the integration policy using the modified JSON - elastic_fleet_integration_update "$INTEGRATION_ID" "$UPDATED_INTEGRATION_POLICY" + if ! elastic_fleet_integration_update "$INTEGRATION_ID" "$UPDATED_INTEGRATION_POLICY"; then + # exit 1 on failure to update fleet integration policies, let salt handle retries + echo "Failed to update $POLICYNAME.." + exit 1 + fi done \ No newline at end of file diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load index 26414a94b..8427b47bc 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load @@ -13,7 +13,7 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then /usr/sbin/so-elastic-fleet-package-upgrade # Second, update Fleet Server policies - /sbin/so-elastic-fleet-integration-policy-elastic-fleet-server + /usr/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server # Third, configure Elastic Defend Integration seperately /usr/sbin/so-elastic-fleet-integration-policy-elastic-defend @@ -25,10 +25,16 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION" if [ -n "$INTEGRATION_ID" ]; then printf "\n\nIntegration $NAME exists - Updating integration\n" - elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION" + if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then + echo -e "\nFailed to update integration for ${INTEGRATION##*/}" + exit 1 + fi else printf "\n\nIntegration does not exist - Creating integration\n" - elastic_fleet_integration_create "@$INTEGRATION" + if ! elastic_fleet_integration_create "@$INTEGRATION"; then + echo -e "\nFailed to create integration for ${INTEGRATION##*/}" + exit 1 + fi fi done @@ -39,10 +45,16 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then elastic_fleet_integration_check "so-grid-nodes_general" "$INTEGRATION" if [ -n "$INTEGRATION_ID" ]; then printf "\n\nIntegration $NAME exists - Updating integration\n" - elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION" + if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then + echo -e "\nFailed to update integration for ${INTEGRATION##*/}" + exit 1 + fi else printf "\n\nIntegration does not exist - Creating integration\n" - elastic_fleet_integration_create "@$INTEGRATION" + if ! elastic_fleet_integration_create "@$INTEGRATION"; then + echo -e "\nFailed to create integration for ${INTEGRATION##*/}" + exit 1 + fi fi done if [[ "$RETURN_CODE" != "1" ]]; then @@ -56,11 +68,17 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then elastic_fleet_integration_check "so-grid-nodes_heavy" "$INTEGRATION" if [ -n "$INTEGRATION_ID" ]; then printf "\n\nIntegration $NAME exists - Updating integration\n" - elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION" + if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then + echo -e "\nFailed to update integration for ${INTEGRATION##*/}" + exit 1 + fi else printf "\n\nIntegration does not exist - Creating integration\n" if [ "$NAME" != "elasticsearch-logs" ]; then - elastic_fleet_integration_create "@$INTEGRATION" + if ! elastic_fleet_integration_create "@$INTEGRATION"; then + echo -e "\nFailed to create integration for ${INTEGRATION##*/}" + exit 1 + fi fi fi done @@ -77,11 +95,17 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then elastic_fleet_integration_check "$FLEET_POLICY" "$INTEGRATION" if [ -n "$INTEGRATION_ID" ]; then printf "\n\nIntegration $NAME exists - Updating integration\n" - elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION" + if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then + echo -e "\nFailed to update integration for ${INTEGRATION##*/}" + exit 1 + fi else printf "\n\nIntegration does not exist - Creating integration\n" if [ "$NAME" != "elasticsearch-logs" ]; then - elastic_fleet_integration_create "@$INTEGRATION" + if ! elastic_fleet_integration_create "@$INTEGRATION"; then + echo -e "\nFailed to create integration for ${INTEGRATION##*/}" + exit 1 + fi fi fi fi diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade index 68a644798..f1154af1e 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade @@ -25,11 +25,16 @@ fi default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %}) for AGENT_POLICY in $agent_policies; do - integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY") + if ! integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY"); then + # this script upgrades default integration packages, exit 1 and let salt handle retrying + exit 1 + fi for INTEGRATION in $integrations; do if ! [[ "$INTEGRATION" == "elastic-defend-endpoints" ]] && ! [[ "$INTEGRATION" == "fleet_server-"* ]]; then # Get package name so we know what package to look for when checking the current and latest available version - PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION") + if ! PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION"); then + exit 1 + fi {%- if not AUTO_UPGRADE_INTEGRATIONS %} if [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then {%- endif %} @@ -48,7 +53,9 @@ for AGENT_POLICY in $agent_policies; do fi # Get integration ID - INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION") + if ! INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION"); then + exit 1 + fi if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then # Dry run of the upgrade @@ -56,14 +63,15 @@ for AGENT_POLICY in $agent_policies; do echo "Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..." echo "Upgrading $INTEGRATION..." echo "Starting dry run..." - DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID") + if ! DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID"); then + exit 1 + fi DRYRUN_ERRORS=$(echo "$DRYRUN_OUTPUT" | jq .[].hasErrors) # If no errors with dry run, proceed with actual upgrade if [[ "$DRYRUN_ERRORS" == "false" ]]; then echo "No errors detected. Proceeding with upgrade..." - elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID" - if [ $? -ne 0 ]; then + if ! elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"; then echo "Error: Upgrade failed for $PACKAGE_NAME with integration ID '$INTEGRATION_ID'." exit 1 fi diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load index 886bbf75c..01777e5da 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load @@ -62,9 +62,17 @@ default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.l in_use_integrations=() for AGENT_POLICY in $agent_policies; do - integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY") + + if ! integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY"); then + # skip the agent policy if we can't get required info, let salt retry. Integrations loaded by this script are non-default integrations. + echo "Skipping $AGENT_POLICY.. " + continue + fi for INTEGRATION in $integrations; do - PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION") + if ! PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION"); then + echo "Not adding $INTEGRATION, couldn't get package name" + continue + fi # non-default integrations that are in-use in any policy if ! [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then in_use_integrations+=("$PACKAGE_NAME") @@ -160,7 +168,11 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then for file in "${pkg_filename}_"*.json; do [ -e "$file" ] || continue - elastic_fleet_bulk_package_install $file >> $BULK_INSTALL_OUTPUT + if ! elastic_fleet_bulk_package_install $file >> $BULK_INSTALL_OUTPUT; then + # integrations loaded my this script are non-essential and shouldn't cause exit, skip them for now next highstate run can retry + echo "Failed to complete a chunk of bulk package installs -- $file " + continue + fi done # cleanup any temp files for chunked package install rm -f ${pkg_filename}_*.json $BULK_INSTALL_PACKAGE_LIST @@ -168,8 +180,9 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then echo "Elastic integrations don't appear to need installation/updating..." fi # Write out file for generating index/component/ilm templates - latest_installed_package_list=$(elastic_fleet_installed_packages) - echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS + if latest_installed_package_list=$(elastic_fleet_installed_packages); then + echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS + fi if retry 3 1 "so-elasticsearch-query / --fail --output /dev/null"; then # Refresh installed component template list latest_component_templates_list=$(so-elasticsearch-query _component_template | jq '.component_templates[] | .name' | jq -s '.') diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load index 819d7ecff..52fa96cd5 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load @@ -10,8 +10,16 @@ {%- for PACKAGE in SUPPORTED_PACKAGES %} echo "Setting up {{ PACKAGE }} package..." -VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") -elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION" +if VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}"); then + if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then + # packages loaded by this script should never fail to install and REQUIRED before an installation of SO can be considered successful + echo -e "\nERROR: Failed to install default integration package -- $PACKAGE $VERSION" + exit 1 + fi +else + echo -e "\nERROR: Failed to get version information for integration $PACKAGE" + exit 1 +fi echo {%- endfor %} echo diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade index a092e3ecb..18211a7c6 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade @@ -10,8 +10,15 @@ {%- for PACKAGE in SUPPORTED_PACKAGES %} echo "Upgrading {{ PACKAGE }} package..." -VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}") -elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION" +if VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then + if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then + # exit 1 on failure to upgrade a default package, allow salt to handle retries + echo -e "\nERROR: Failed to upgrade $PACKAGE to version: $VERSION" + exit 1 + fi +else + echo -e "\nERROR: Failed to get version information for integration $PACKAGE" +fi echo {%- endfor %} echo diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index deb16dadf..ee74d1056 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -23,18 +23,17 @@ if [[ "$RETURN_CODE" != "0" ]]; then exit 1 fi -ALIASES=".fleet-servers .fleet-policies-leader .fleet-policies .fleet-agents .fleet-artifacts .fleet-enrollment-api-keys .kibana_ingest" -for ALIAS in ${ALIASES} -do +ALIASES=(.fleet-servers .fleet-policies-leader .fleet-policies .fleet-agents .fleet-artifacts .fleet-enrollment-api-keys .kibana_ingest) +for ALIAS in "${ALIASES[@]}"; do # Get all concrete indices from alias - INDXS=$(curl -K /opt/so/conf/kibana/curl.config -s -k -L -H "Content-Type: application/json" "https://localhost:9200/_resolve/index/${ALIAS}" | jq -r '.aliases[].indices[]') - - # Delete all resolved indices - for INDX in ${INDXS} - do + if INDXS_RAW=$(curl -sK /opt/so/conf/kibana/curl.config -s -k -L -H "Content-Type: application/json" "https://localhost:9200/_resolve/index/${ALIAS}" --fail 2>/dev/null); then + INDXS=$(echo "$INDXS_RAW" | jq -r '.aliases[].indices[]') + # Delete all resolved indices + for INDX in ${INDXS}; do status "Deleting $INDX" curl -K /opt/so/conf/kibana/curl.config -s -k -L -H "Content-Type: application/json" "https://localhost:9200/${INDX}" -XDELETE - done + done + fi done # Restarting Kibana... @@ -51,22 +50,54 @@ if [[ "$RETURN_CODE" != "0" ]]; then fi printf "\n### Create ES Token ###\n" -ESTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/service_tokens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq -r .value) +if ESTOKEN_RAW=$(fleet_api "service_tokens" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then + ESTOKEN=$(echo "$ESTOKEN_RAW" | jq -r .value) +else + echo -e "\nFailed to create ES token..." + exit 1 +fi ### Create Outputs, Fleet Policy and Fleet URLs ### # Create the Manager Elasticsearch Output first and set it as the default output printf "\nAdd Manager Elasticsearch Output...\n" -ESCACRT=$(openssl x509 -in $INTCA) -JSON_STRING=$( jq -n \ - --arg ESCACRT "$ESCACRT" \ - '{"name":"so-manager_elasticsearch","id":"so-manager_elasticsearch","type":"elasticsearch","hosts":["https://{{ GLOBALS.manager_ip }}:9200","https://{{ GLOBALS.manager }}:9200"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate_authorities": [$ESCACRT]}}' ) -curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" +ESCACRT=$(openssl x509 -in "$INTCA" -outform DER | sha256sum | cut -d' ' -f1 | tr '[:lower:]' '[:upper:]') +JSON_STRING=$(jq -n \ + --arg ESCACRT "$ESCACRT" \ + '{"name":"so-manager_elasticsearch","id":"so-manager_elasticsearch","type":"elasticsearch","hosts":["https://{{ GLOBALS.manager_ip }}:9200","https://{{ GLOBALS.manager }}:9200"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ca_trusted_fingerprint": $ESCACRT}') + +if ! fleet_api "outputs" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then + echo -e "\nFailed to create so-elasticsearch_manager policy..." + exit 1 +fi printf "\n\n" +# At this point there should only be two policies. fleet-default-output & so-manager_elasticsearch +status "Verifying so-manager_elasticsearch policy is configured as the current default" + +# Grab the fleet-default-output policy instead of so-manager_elasticsearch, because a weird state can exist where both fleet-default-output & so-elasticsearch_manager can be set as the active default output for logs / metrics. Resulting in logs not ingesting on import/eval nodes +if DEFAULTPOLICY=$(fleet_api "outputs/fleet-default-output"); then + fleet_default=$(echo "$DEFAULTPOLICY" | jq -er '.item.is_default') + fleet_default_monitoring=$(echo "$DEFAULTPOLICY" | jq -er '.item.is_default_monitoring') +# Check that fleet-default-output isn't configured as a default for anything ( both variables return false ) + if [[ ! $fleet_default ]] && [[ ! $fleet_default_monitoring ]]; then + echo -e "\nso-manager_elasticsearch is configured as the current default policy..." + else + echo -e "\nVerification of so-manager_elasticsearch policy failed... The default 'fleet-default-output' output is still active..." + exit 1 + fi +else + # fleet-output-policy is created automatically by fleet when started. Should always exist on any installation type + echo -e "\nDefault fleet-default-output policy doesn't exist...\n" + exit 1 +fi + # Create the Manager Fleet Server Host Agent Policy # This has to be done while the Elasticsearch Output is set to the default Output printf "Create Manager Fleet Server Policy...\n" -elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "false" "120" +if ! elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "false" "120"; then + echo -e "\n Failed to create Manager fleet server policy..." + exit 1 +fi # Modify the default integration policy to update the policy_id with the correct naming UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "FleetServer_{{ GLOBALS.hostname }}" --arg name "fleet_server-{{ GLOBALS.hostname }}" ' @@ -74,7 +105,10 @@ UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "FleetServer_{{ GLOBALS.hostname .name = $name' /opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json) # Add the Fleet Server Integration to the new Fleet Policy -elastic_fleet_integration_create "$UPDATED_INTEGRATION_POLICY" +if ! elastic_fleet_integration_create "$UPDATED_INTEGRATION_POLICY"; then + echo -e "\nFailed to create Fleet server integration for Manager.." + exit 1 +fi # Now we can create the Logstash Output and set it to to be the default Output printf "\n\nCreate Logstash Output Config if node is not an Import or Eval install\n" @@ -88,7 +122,10 @@ JSON_STRING=$( jq -n \ --arg LOGSTASHCA "$LOGSTASHCA" \ '{"name":"grid-logstash","is_default":true,"is_default_monitoring":true,"id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055", "{{ GLOBALS.manager }}:5055"],"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]},"proxy_id":null}' ) -curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" +if ! fleet_api "outputs" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then + echo -e "\nFailed to create logstash fleet output" + exit 1 +fi printf "\n\n" {%- endif %} @@ -106,7 +143,10 @@ else fi ## This array replaces whatever URLs are currently configured -curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/fleet_server_hosts" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" +if ! fleet_api "fleet_server_hosts" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then + echo -e "\nFailed to add manager fleet URL" + exit 1 +fi printf "\n\n" ### Create Policies & Associated Integration Configuration ### @@ -117,13 +157,22 @@ printf "\n\n" /usr/sbin/so-elasticsearch-templates-load # Initial Endpoints Policy -elastic_fleet_policy_create "endpoints-initial" "Initial Endpoint Policy" "false" "1209600" +if ! elastic_fleet_policy_create "endpoints-initial" "Initial Endpoint Policy" "false" "1209600"; then + echo -e "\nFailed to create endpoints-initial policy..." + exit 1 +fi # Grid Nodes - General Policy -elastic_fleet_policy_create "so-grid-nodes_general" "SO Grid Nodes - General Purpose" "false" "1209600" +if ! elastic_fleet_policy_create "so-grid-nodes_general" "SO Grid Nodes - General Purpose" "false" "1209600"; then + echo -e "\nFailed to create so-grid-nodes_general policy..." + exit 1 +fi # Grid Nodes - Heavy Node Policy -elastic_fleet_policy_create "so-grid-nodes_heavy" "SO Grid Nodes - Heavy Node" "false" "1209600" +if ! elastic_fleet_policy_create "so-grid-nodes_heavy" "SO Grid Nodes - Heavy Node" "false" "1209600"; then + echo -e "\nFailed to create so-grid-nodes_heavy policy..." + exit 1 +fi # Load Integrations for default policies so-elastic-fleet-integration-policy-load @@ -135,14 +184,34 @@ JSON_STRING=$( jq -n \ '{"name":$NAME,"host":$URL,"is_default":true}' ) -curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_download_sources" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" +if ! fleet_api "agent_download_sources" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then + echo -e "\nFailed to update Elastic Agent artifact URL" + exit 1 +fi ### Finalization ### # Query for Enrollment Tokens for default policies -ENDPOINTSENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key') -GRIDNODESENROLLMENTOKENGENERAL=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_general")) | .api_key') -GRIDNODESENROLLMENTOKENHEAVY=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_heavy")) | .api_key') +if ENDPOINTSENROLLMENTOKEN_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then + ENDPOINTSENROLLMENTOKEN=$(echo "$ENDPOINTSENROLLMENTOKEN_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key') +else + echo -e "\nFailed to query for Endpoints enrollment token" + exit 1 +fi + +if GRIDNODESENROLLMENTOKENGENERAL_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then + GRIDNODESENROLLMENTOKENGENERAL=$(echo "$GRIDNODESENROLLMENTOKENGENERAL_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_general")) | .api_key') +else + echo -e "\nFailed to query for Grid nodes - General enrollment token" + exit 1 +fi + +if GRIDNODESENROLLMENTOKENHEAVY_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then + GRIDNODESENROLLMENTOKENHEAVY=$(echo "$GRIDNODESENROLLMENTOKENHEAVY_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_heavy")) | .api_key') +else + echo -e "\nFailed to query for Grid nodes - Heavy enrollment token" + exit 1 +fi # Store needed data in minion pillar pillar_file=/opt/so/saltstack/local/pillar/minions/{{ GLOBALS.minion_id }}.sls diff --git a/salt/kratos/enabled.sls b/salt/kratos/enabled.sls index 31097ccf4..f0345edec 100644 --- a/salt/kratos/enabled.sls +++ b/salt/kratos/enabled.sls @@ -54,6 +54,9 @@ so-kratos: - file: kratosconfig - file: kratoslogdir - file: kratosdir + - retry: + attempts: 10 + interval: 10 delete_so-kratos_so-status.disabled: file.uncomment: diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 86595c162..e49be133f 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -450,6 +450,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == 2.4.150 ]] && post_to_2.4.160 [[ "$POSTVERSION" == 2.4.160 ]] && post_to_2.4.170 [[ "$POSTVERSION" == 2.4.170 ]] && post_to_2.4.180 + [[ "$POSTVERSION" == 2.4.180 ]] && post_to_2.4.190 true } @@ -608,6 +609,15 @@ post_to_2.4.180() { POSTVERSION=2.4.180 } +post_to_2.4.190() { + # Only need to update import / eval nodes + if [[ "$MINION_ROLE" == "import" ]] || [[ "$MINION_ROLE" == "eval" ]]; then + update_import_fleet_output + fi + + POSTVERSION=2.4.190 +} + repo_sync() { echo "Sync the local repo." su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync." @@ -870,6 +880,11 @@ up_to_2.4.180() { INSTALLEDVERSION=2.4.180 } +up_to_2.4.190() { + echo "Nothing to do for 2.4.190" + INSTALLEDVERSION=2.4.190 +} + add_hydra_pillars() { mkdir -p /opt/so/saltstack/local/pillar/hydra touch /opt/so/saltstack/local/pillar/hydra/soc_hydra.sls @@ -1143,6 +1158,19 @@ update_elasticsearch_index_settings() { done } +update_import_fleet_output() { + if output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" --retry 3 --fail 2>/dev/null); then + # Update the current config of so-manager_elasticsearch output policy in place (leaving any customizations like having changed the preset value from 'balanced' to 'performance') + CAFINGERPRINT=$(openssl x509 -in /etc/pki/tls/certs/intca.crt -outform DER | sha256sum | cut -d' ' -f1 | tr '[:lower:]' '[:upper:]') + updated_policy=$(jq --arg CAFINGERPRINT "$CAFINGERPRINT" '.item | (del(.id) | .ca_trusted_fingerprint = $CAFINGERPRINT)' <<< "$output") + if curl -sK /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" -XPUT -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$updated_policy" --retry 3 --fail 2>/dev/null; then + echo "Successfully updated so-manager_elasticsearch fleet output policy" + else + fail "Failed to update so-manager_elasticsearch fleet output policy" + fi + fi +} + update_salt_mine() { echo "Populating the mine with mine_functions for each host." set +e @@ -1359,6 +1387,7 @@ main() { fi set_minionid + MINION_ROLE=$(lookup_role) echo "Found that Security Onion $INSTALLEDVERSION is currently installed." echo "" if [[ $is_airgap -eq 0 ]]; then @@ -1401,7 +1430,7 @@ main() { if [ "$is_hotfix" == "true" ]; then echo "Applying $HOTFIXVERSION hotfix" # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars - if [[ ! "$MINIONID" =~ "_import" ]]; then + if [[ ! "$MINION_ROLE" == "import" ]]; then backup_old_states_pillars fi copy_new_files @@ -1464,7 +1493,7 @@ main() { fi # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars - if [[ ! "$MINIONID" =~ "_import" ]]; then + if [[ ! "$MINION_ROLE" == "import" ]]; then echo "" echo "Creating snapshots of default and local Salt states and pillars and saving to /nsm/backup/" backup_old_states_pillars diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 81baa100a..62b7f1b18 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -4,7 +4,10 @@ Elastic License 2.0. #} {% set role = salt['grains.get']('role', '') %} -{% if role in ['so-hypervisor','so-managerhype'] %} +{# We are using usebr0 mostly for setup of the so-managerhype node and controlling when we use br0 vs the physical interface #} +{% set usebr0 = salt['pillar.get']('usebr0', True) %} + +{% if role in ['so-hypervisor','so-managerhype'] and usebr0 %} {% set interface = 'br0' %} {% else %} {% set interface = pillar.host.mainint %} diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 58b3a3827..6caeddbe3 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -2545,7 +2545,7 @@ soc: level: 'high' # info | low | medium | high | critical assistant: enabled: false - investigationPrompt: Investigate Alert ID {socid} + investigationPrompt: Investigate Alert ID {socId} contextLimitSmall: 200000 contextLimitLarge: 1000000 thresholdColorRatioLow: 0.5 diff --git a/setup/so-functions b/setup/so-functions index 00f2e46c1..0d7890d17 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -541,8 +541,15 @@ configure_minion() { "log_file: /opt/so/log/salt/minion"\ "#startup_states: highstate" >> "$minion_config" - info "Running: salt-call state.apply salt.mine_functions --local --file-root=../salt/ -l info pillar='{"host": {"mainint": "$MNIC"}}'" - salt-call state.apply salt.mine_functions --local --file-root=../salt/ -l info pillar="{'host': {'mainint': $MNIC}}" + # At the time the so-managerhype node does not yet have the bridge configured. + # The so-hypervisor node doesn't either, but it doesn't cause issues here. + local usebr0=false + if [ "$minion_type" == 'hypervisor' ]; then + usebr0=true + fi + local pillar_json="{\"host\": {\"mainint\": \"$MNIC\"}, \"usebr0\": $usebr0}" + info "Running: salt-call state.apply salt.mine_functions --local --file-root=../salt/ -l info pillar='$pillar_json'" + salt-call state.apply salt.mine_functions --local --file-root=../salt/ -l info pillar="$pillar_json" { logCmd "systemctl enable salt-minion"; @@ -1195,9 +1202,9 @@ hypervisor_local_states() { logCmd "salt-call state.apply libvirt.64962 --local --file-root=../salt/ -l info queue=True" info "Setting up bridge for $MNIC" salt-call state.apply libvirt.bridge --local --file-root=../salt/ -l info pillar='{"host": {"mainint": "'$MNIC'"}}' queue=True - if [ $is_managerhype ]; then - logCmd "salt-call state.apply salt.minion queue=True" - fi + #if [ $is_managerhype ]; then + # logCmd "salt-call state.apply salt.minion queue=True" + #fi fi } diff --git a/setup/so-setup b/setup/so-setup index 347a7165c..ab055fd2d 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -762,6 +762,7 @@ if ! [[ -f $install_opt_file ]]; then fi logCmd "salt-call state.apply common.packages" logCmd "salt-call state.apply common" + hypervisor_local_states # this will apply the salt.minion state first since salt.master includes salt.minion logCmd "salt-call state.apply salt.master" # wait here until we get a response from the salt-master since it may have just restarted @@ -826,7 +827,6 @@ if ! [[ -f $install_opt_file ]]; then checkin_at_boot set_initial_firewall_access logCmd "salt-call schedule.enable -linfo --local" - hypervisor_local_states verify_setup else touch /root/accept_changes