Merge pull request #16378 from Security-Onion-Solutions/reyesj2/fix-16369

surface ilm policy errors in so-elasticsearch-ilm-policy-load output

.github/DISCUSSION_TEMPLATE/3-0.yml

@@ -14,6 +14,7 @@ body:
         - 3.2.0
         - 3.3.0
         - 3.4.0
+        - 3.5.0
         - Other (please provide detail below)
     validations:
       required: true

VERSION

@@ -1 +1 @@
-3.4.0
+3.5.0

salt/desktop/packages.sls

@@ -438,6 +438,10 @@ install_networkminer:
   pkg.latest:
     - name: securityonion-networkminer
 
+install_so_crates:
+  pkg.latest:
+    - name: securityonion-so-crates
+
 {% else %}
 
 desktop_packages_os_fail:

salt/elasticsearch/cluster.sls

@@ -107,6 +107,12 @@ so-elasticsearch-system-indices-patch:
       - file: so-elasticsearch-system-indices-patch-script
 {%  endif %}
 
+so-elasticsearch-ilm-policy-reload:
+  file.absent:
+    - name: /opt/so/state/esilm.txt
+    - onchanges:
+      - file: so-elasticsearch-ilm-policy-load-script
+
 # heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)
 so-elasticsearch-ilm-policy-load:
   cmd.run:
@@ -115,8 +121,9 @@ so-elasticsearch-ilm-policy-load:
     - require:
       - docker_container: so-elasticsearch
       - file: so-elasticsearch-ilm-policy-load-script
-    - onchanges:
+      - http: wait_for_so-elasticsearch
-      - file: so-elasticsearch-ilm-policy-load-script
+      - file: so-elasticsearch-ilm-policy-reload
+    - unless: test -f /opt/so/state/esilm.txt
 
 so-elasticsearch-templates-reload:
   file.absent:

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load

@@ -7,11 +7,17 @@
 . /usr/sbin/so-common
 
 MAX_JOBS=${MAX_ILM_JOBS:-10}
+ILM_STATEFILE=/opt/so/state/esilm.txt
+rm -f "$ILM_STATEFILE"
 
+check_elasticsearch_responsive
+
+ILM_TMP_DIR=$(mktemp -d)
+trap 'rm -rf "$ILM_TMP_DIR"' EXIT
 # Lock used to serialize block writes so concurrent jobs never interleave their output.
-ILM_OUTPUT_LOCK=$(mktemp)
+ILM_OUTPUT_LOCK="$ILM_TMP_DIR/output.lock"
-ILM_FAIL_FILE=$(mktemp)
+ILM_FAIL_FILE="$ILM_TMP_DIR/failed"
-trap 'rm -f "$ILM_OUTPUT_LOCK" "$ILM_FAIL_FILE"' EXIT
+touch "$ILM_FAIL_FILE"
 
 # Policies are loaded concurrently (up to MAX_JOBS at a time) for speed. Each policy's block is
 # printed the moment its curl returns, so output appears in COMPLETION ORDER, not the order
@@ -20,13 +26,30 @@ echo "Loading ILM policies concurrently; output below appears in completion orde
 echo
 
 put_policy() {
-  local desc="$1" policyname="$2" data="$3" result rc=0
+  local desc="$1" policyname="$2" data="$3" rc=1 attempt out http_code body result reason tmpdata
-  if ! result=$(curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L --fail \
+  tmpdata=$(mktemp "$ILM_TMP_DIR/${policyname}.XXXXXX.json")
-    -X PUT "https://localhost:9200/_ilm/policy/${policyname}" \
+  printf '%s' "$data" >"$tmpdata"
-    -H 'Content-Type: application/json' -d"${data}" 2>&1); then
+
-    rc=1
+  for attempt in 1 2 3; do
-  elif ! jq -e '.acknowledged == true' <<<"$result" >/dev/null 2>&1; then
+    out=$(so-elasticsearch-query "_ilm/policy/${policyname}" -d@"${tmpdata}" -XPUT -w '\n%{http_code}' 2>/dev/null)
-    rc=1
+    http_code=${out##*$'\n'} body=${out%$'\n'*}
+
+    if [[ "$http_code" == 200 ]] && jq -e '.acknowledged == true' <<<"$body" >/dev/null 2>&1; then
+      rc=0
+      break
+    fi
+    # 429 = ES rejected/throttled (may succeed later); other 4xx will not
+    [[ "$http_code" == 4?? && "$http_code" != 429 ]] && break
+    (( attempt < 3 )) && sleep 3
+  done
+
+  if (( rc == 0 )); then
+    result='{"acknowledged":true}'
+  else
+    result=${body:-"HTTP ${http_code:-000}"}
+    # flatten tabs/newlines so the TAB fail-file stays one record
+    reason=$(jq -r '(if (.error|type) == "object" then (.error.reason // .error.type) elif (.error|type) == "string" then .error else empty end) // empty | tostring | gsub("[\t\r\n]+"; " ")' <<<"$body" 2>/dev/null)
+    reason="HTTP ${http_code:-000}${reason:+: ${reason}}"
   fi
 
   # curl above ran in parallel; serialize just this block write so concurrent jobs never interleave.
@@ -34,7 +57,7 @@ put_policy() {
     flock 200
     printf 'Setting up %s policy...\n%s\n\n' "${desc}" "${result}"
     if (( rc != 0 )); then
-      printf '%s\n' "${policyname}" >>"$ILM_FAIL_FILE"
+      printf '%s\t%s\n' "${policyname}" "${reason}" >>"$ILM_FAIL_FILE"
     fi
   } 200>>"${ILM_OUTPUT_LOCK}"
 
@@ -82,10 +105,15 @@ wait || true
 
 if [[ -s "$ILM_FAIL_FILE" ]]; then
   echo "ERROR: Failed to load ILM policy(s):"
-  while read -r POLICY; do
+  while IFS=$'\t' read -r POLICY REASON; do
-    echo " - $POLICY"
+    if [[ -n "$REASON" ]]; then
+      echo " - ${POLICY}: ${REASON}"
+    else
+      echo " - ${POLICY}"
+    fi
   done < "$ILM_FAIL_FILE"
   exit 1
 else
   echo "Successfully loaded all ILM policies."
+  touch "$ILM_STATEFILE"
 fi

salt/manager/tools/sbin/soup

@@ -121,9 +121,6 @@ check_err() {
       161)
         echo 'Required intermediate Elasticsearch upgrade not complete'
       ;;
-      162)
-        echo 'One or more Elastic Agent nodes do not support the x86-64-v3 CPU instruction set'
-      ;;
       170)
         echo "Intermediate upgrade completed successfully to $next_step_so_version, but next soup to Security Onion $originally_requested_so_version could not be started automatically."
         echo "Start soup again manually to continue the upgrade to Security Onion $originally_requested_so_version."
@@ -351,83 +348,6 @@ check_cluster_health() {
   exit 0
 }
 
-no_soup_for_you() {
-  echo ""
-  echo "No soup for you!"
-  exit 162
-}
-
-check_cpu_compatibility() {
-  # Roles running a container built from the so-elastic-agent image; mirrors the
-  # elasticagent and elasticfleet entries in salt/reactor/pillar_push_map.yaml.
-  local cpu_target='G@role:so-heavynode or G@role:so-eval or G@role:so-fleet or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone'
-  local expected_nodes cpu_results node result confirm
-  local -a unsupported=() offline=()
-
-  echo "Checking that Elastic Agent nodes support the x86-64-v3 CPU instruction set now required by Elastic."
-
-  if [[ "$SKIP_CPU_CHECK" == "true" ]]; then
-    printf "\nSkipping the x86-64-v3 CPU check because --skip-cpu-check was specified.\n\n"
-    return
-  fi
-
-  # Nodes that never answer are absent from the results, so diff against who should have.
-  expected_nodes=$(salt -C "$cpu_target" --preview-target --out=json 2>/dev/null | jq -r '.[]?') || true
-  if [[ -z "$expected_nodes" ]]; then
-    printf "\nCould not determine which nodes run the Elastic Agent, so the x86-64-v3 CPU check cannot run.\n"
-    no_soup_for_you
-  fi
-
-  cpu_results=$(salt -t 30 -C "$cpu_target" cmd.run "/lib64/ld-linux-x86-64.so.2 --help | grep x86-64-v3" --out=json 2>/dev/null) || true
-
-  while IFS= read -r node; do
-    [[ -z "$node" ]] && continue
-    result=$(jq -r --arg node "$node" '.[$node] // empty' <<< "$cpu_results" 2>/dev/null)
-    if [[ -z "$result" || "$result" == *"did not return"* ]]; then
-      offline+=("$node")
-    elif [[ "$result" != *"x86-64-v3 (supported"* ]]; then
-      # glibc appends "(supported, searched)" only when supported; the open paren keeps
-      # this from matching a future "(unsupported".
-      unsupported+=("$node")
-    fi
-  done <<< "$expected_nodes"
-
-  if [[ ${#unsupported[@]} -eq 0 && ${#offline[@]} -eq 0 ]]; then
-    printf "\nAll Elastic Agent nodes support x86-64-v3. We can proceed with SOUP.\n\n"
-    return
-  fi
-
-  echo ""
-  if [[ ${#unsupported[@]} -gt 0 ]]; then
-    echo "The following node(s) do NOT support the x86-64-v3 CPU instruction set:"
-    printf '  %s\n' "${unsupported[@]}"
-    echo ""
-    echo "Upstream Elastic now builds its binaries for x86-64-v3, so these nodes can no"
-    echo "longer run Elastic. Upgrading them WILL BREAK them."
-    echo ""
-  fi
-  if [[ ${#offline[@]} -gt 0 ]]; then
-    echo "The following node(s) did not respond and could not be checked:"
-    printf '  %s\n' "${offline[@]}"
-    echo ""
-    echo "These nodes are offline, so we cannot confirm they support x86-64-v3, which"
-    echo "upstream Elastic now requires."
-    echo ""
-  fi
-
-  if [[ -n $UNATTENDED ]]; then
-    echo "Unattended mode cannot prompt for an override. Re-run soup interactively, or pass --skip-cpu-check to bypass this check."
-    no_soup_for_you
-  fi
-
-  read -rp "Type 'override' to continue anyway, or press Enter to exit: " confirm
-  if [[ "${confirm,,}" == "override" ]]; then
-    printf "\nOverride accepted. Continuing at your own risk.\n\n"
-  else
-    no_soup_for_you
-  fi
-}
-
 check_fleet_server() {
   echo "Checking that Elastic Fleet Server is responding."
   # Modeled on the wait_for_so-elastic-fleet state check in elasticfleet/enabled.sls,
@@ -2168,8 +2088,6 @@ main() {
   echo "Let's see if we need to update Security Onion."
   upgrade_check
 
-  check_cpu_compatibility
-
   upgrade_space
 
   echo "Verifying Elasticsearch version compatibility across the grid before upgrading."
@@ -2448,17 +2366,6 @@ fi
   echo "### soup has been served at $(date) ###"
 }
 
-SKIP_CPU_CHECK=false
-declare -a SOUP_ARGS=()
-for arg in "$@"; do
-  if [[ "$arg" == "--skip-cpu-check" ]]; then
-    SKIP_CPU_CHECK=true
-  else
-    SOUP_ARGS+=("$arg")
-  fi
-done
-set -- "${SOUP_ARGS[@]}"
-
 while getopts ":b:f:y" opt; do
   case ${opt} in
     b )
@@ -2482,7 +2389,7 @@ while getopts ":b:f:y" opt; do
       ISOLOC="$OPTARG"
     ;;
     \? )
-      echo "Usage: soup [-b] [-y] [-f <iso location>] [--skip-cpu-check]"
+      echo "Usage: soup [-b] [-y] [-f <iso location>]"
       exit 1
     ;;
     : )

setup/so-functions

@@ -976,15 +976,39 @@ docker_seed_update() {
 	set_progress_str "$docker_seed_update_percent" "Downloading $name"
 }
 
+registry_tar_has_soversion() {
+	local tarfile=$1
+	local version=$2
+
+	[ -n "$version" ] || return 1
+	[ -f "$tarfile" ] || return 1
+	# grep -q in a pipe SIGPIPEs tar; under pipefail a match would look like failure.
+	grep -qF "/_manifests/tags/${version}/" < <(tar -tf "$tarfile" 2>/dev/null)
+}
+
 docker_seed_registry() {
 	local VERSION="$SOVERSION"
+	local REGISTRY_TAR=/nsm/docker-registry/docker/registry.tar
 
-	if [ -f /nsm/docker-registry/docker/registry.tar ]; then
+	if [ -f "$REGISTRY_TAR" ] && registry_tar_has_soversion "$REGISTRY_TAR" "$VERSION"; then
-		logCmd "tar xvf /nsm/docker-registry/docker/registry.tar -C /nsm/docker-registry/docker"
+		info "Extracting $REGISTRY_TAR"
-		logCmd "rm /nsm/docker-registry/docker/registry.tar"
+		if ! tar xvf "$REGISTRY_TAR" -C /nsm/docker-registry/docker >> "$setup_log" 2>&1; then
+			error "Failed to extract $REGISTRY_TAR"
+			return 1
+		fi
+		logCmd "rm -f $REGISTRY_TAR"
+		return 0
+	elif [ -f "$REGISTRY_TAR" ]; then
+		info "Not using $REGISTRY_TAR; it has no images tagged '${VERSION}'"
+		if [[ "$is_airgap" == true ]]; then
+			error "Airgap install: $REGISTRY_TAR does not match Security Onion version '${VERSION}'. Use the ISO for this version."
+			return 1
+		fi
 	elif [[ -d /nsm/docker-registry/docker/registry && ( -f /etc/SOCLOUD || "$is_airgap" == true ) ]]; then
 		echo "Using existing docker registry content"
-	else
+		return 0
+	fi
+
 	if [ "$install_type" == 'IMPORT' ]; then
 		container_list 'so-import'	
 	else
@@ -996,7 +1020,6 @@ docker_seed_registry() {
 	update_docker_containers 'netinstall' '' 'docker_seed_update' '/dev/stdout' 2>&1 | tee -a "$setup_log"
         # Use pipe exit status of 'update_docker_containers' for return code
 	return ${PIPESTATUS[0]}
-	fi
 }
 
 elasticfleet_pillar() {