Merge pull request #16366 from Security-Onion-Solutions/3/dev

3.4.0

.github/DISCUSSION_TEMPLATE/3-0.yml

@@ -14,7 +14,6 @@ body:
         - 3.2.0
         - 3.3.0
         - 3.4.0
-        - 3.5.0
         - Other (please provide detail below)
     validations:
       required: true

VERSION

@@ -1 +1 @@
-3.5.0
+3.4.0

salt/desktop/packages.sls

@@ -438,10 +438,6 @@ install_networkminer:
   pkg.latest:
     - name: securityonion-networkminer
 
-install_so_crates:
-  pkg.latest:
-    - name: securityonion-so-crates
-
 {% else %}
 
 desktop_packages_os_fail:

salt/elasticsearch/cluster.sls

@@ -107,12 +107,6 @@ so-elasticsearch-system-indices-patch:
       - file: so-elasticsearch-system-indices-patch-script
 {%  endif %}
 
-so-elasticsearch-ilm-policy-reload:
-  file.absent:
-    - name: /opt/so/state/esilm.txt
-    - onchanges:
-      - file: so-elasticsearch-ilm-policy-load-script
-
 # heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)
 so-elasticsearch-ilm-policy-load:
   cmd.run:
@@ -121,9 +115,8 @@ so-elasticsearch-ilm-policy-load:
     - require:
       - docker_container: so-elasticsearch
       - file: so-elasticsearch-ilm-policy-load-script
-      - http: wait_for_so-elasticsearch
+    - onchanges:
-      - file: so-elasticsearch-ilm-policy-reload
+      - file: so-elasticsearch-ilm-policy-load-script
-    - unless: test -f /opt/so/state/esilm.txt
 
 so-elasticsearch-templates-reload:
   file.absent:

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load

@@ -7,17 +7,11 @@
 . /usr/sbin/so-common
 
 MAX_JOBS=${MAX_ILM_JOBS:-10}
-ILM_STATEFILE=/opt/so/state/esilm.txt
-rm -f "$ILM_STATEFILE"
 
-check_elasticsearch_responsive
-
-ILM_TMP_DIR=$(mktemp -d)
-trap 'rm -rf "$ILM_TMP_DIR"' EXIT
 # Lock used to serialize block writes so concurrent jobs never interleave their output.
-ILM_OUTPUT_LOCK="$ILM_TMP_DIR/output.lock"
+ILM_OUTPUT_LOCK=$(mktemp)
-ILM_FAIL_FILE="$ILM_TMP_DIR/failed"
+ILM_FAIL_FILE=$(mktemp)
-touch "$ILM_FAIL_FILE"
+trap 'rm -f "$ILM_OUTPUT_LOCK" "$ILM_FAIL_FILE"' EXIT
 
 # Policies are loaded concurrently (up to MAX_JOBS at a time) for speed. Each policy's block is
 # printed the moment its curl returns, so output appears in COMPLETION ORDER, not the order
@@ -26,30 +20,13 @@ echo "Loading ILM policies concurrently; output below appears in completion orde
 echo
 
 put_policy() {
-  local desc="$1" policyname="$2" data="$3" rc=1 attempt out http_code body result reason tmpdata
+  local desc="$1" policyname="$2" data="$3" result rc=0
-  tmpdata=$(mktemp "$ILM_TMP_DIR/${policyname}.XXXXXX.json")
+  if ! result=$(curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L --fail \
-  printf '%s' "$data" >"$tmpdata"
+    -X PUT "https://localhost:9200/_ilm/policy/${policyname}" \
-
+    -H 'Content-Type: application/json' -d"${data}" 2>&1); then
-  for attempt in 1 2 3; do
+    rc=1
-    out=$(so-elasticsearch-query "_ilm/policy/${policyname}" -d@"${tmpdata}" -XPUT -w '\n%{http_code}' 2>/dev/null)
+  elif ! jq -e '.acknowledged == true' <<<"$result" >/dev/null 2>&1; then
-    http_code=${out##*$'\n'} body=${out%$'\n'*}
+    rc=1
-
-    if [[ "$http_code" == 200 ]] && jq -e '.acknowledged == true' <<<"$body" >/dev/null 2>&1; then
-      rc=0
-      break
-    fi
-    # 429 = ES rejected/throttled (may succeed later); other 4xx will not
-    [[ "$http_code" == 4?? && "$http_code" != 429 ]] && break
-    (( attempt < 3 )) && sleep 3
-  done
-
-  if (( rc == 0 )); then
-    result='{"acknowledged":true}'
-  else
-    result=${body:-"HTTP ${http_code:-000}"}
-    # flatten tabs/newlines so the TAB fail-file stays one record
-    reason=$(jq -r '(if (.error|type) == "object" then (.error.reason // .error.type) elif (.error|type) == "string" then .error else empty end) // empty | tostring | gsub("[\t\r\n]+"; " ")' <<<"$body" 2>/dev/null)
-    reason="HTTP ${http_code:-000}${reason:+: ${reason}}"
   fi
 
   # curl above ran in parallel; serialize just this block write so concurrent jobs never interleave.
@@ -57,7 +34,7 @@ put_policy() {
     flock 200
     printf 'Setting up %s policy...\n%s\n\n' "${desc}" "${result}"
     if (( rc != 0 )); then
-      printf '%s\t%s\n' "${policyname}" "${reason}" >>"$ILM_FAIL_FILE"
+      printf '%s\n' "${policyname}" >>"$ILM_FAIL_FILE"
     fi
   } 200>>"${ILM_OUTPUT_LOCK}"
 
@@ -105,15 +82,10 @@ wait || true
 
 if [[ -s "$ILM_FAIL_FILE" ]]; then
   echo "ERROR: Failed to load ILM policy(s):"
-  while IFS=$'\t' read -r POLICY REASON; do
+  while read -r POLICY; do
-    if [[ -n "$REASON" ]]; then
+    echo " - $POLICY"
-      echo " - ${POLICY}: ${REASON}"
-    else
-      echo " - ${POLICY}"
-    fi
   done < "$ILM_FAIL_FILE"
   exit 1
 else
   echo "Successfully loaded all ILM policies."
-  touch "$ILM_STATEFILE"
 fi

salt/manager/tools/sbin/soup

@@ -121,6 +121,9 @@ check_err() {
       161)
         echo 'Required intermediate Elasticsearch upgrade not complete'
       ;;
+      162)
+        echo 'One or more Elastic Agent nodes do not support the x86-64-v3 CPU instruction set'
+      ;;
       170)
         echo "Intermediate upgrade completed successfully to $next_step_so_version, but next soup to Security Onion $originally_requested_so_version could not be started automatically."
         echo "Start soup again manually to continue the upgrade to Security Onion $originally_requested_so_version."
@@ -348,6 +351,83 @@ check_cluster_health() {
   exit 0
 }
 
+no_soup_for_you() {
+  echo ""
+  echo "No soup for you!"
+  exit 162
+}
+
+check_cpu_compatibility() {
+  # Roles running a container built from the so-elastic-agent image; mirrors the
+  # elasticagent and elasticfleet entries in salt/reactor/pillar_push_map.yaml.
+  local cpu_target='G@role:so-heavynode or G@role:so-eval or G@role:so-fleet or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone'
+  local expected_nodes cpu_results node result confirm
+  local -a unsupported=() offline=()
+
+  echo "Checking that Elastic Agent nodes support the x86-64-v3 CPU instruction set now required by Elastic."
+
+  if [[ "$SKIP_CPU_CHECK" == "true" ]]; then
+    printf "\nSkipping the x86-64-v3 CPU check because --skip-cpu-check was specified.\n\n"
+    return
+  fi
+
+  # Nodes that never answer are absent from the results, so diff against who should have.
+  expected_nodes=$(salt -C "$cpu_target" --preview-target --out=json 2>/dev/null | jq -r '.[]?') || true
+  if [[ -z "$expected_nodes" ]]; then
+    printf "\nCould not determine which nodes run the Elastic Agent, so the x86-64-v3 CPU check cannot run.\n"
+    no_soup_for_you
+  fi
+
+  cpu_results=$(salt -t 30 -C "$cpu_target" cmd.run "/lib64/ld-linux-x86-64.so.2 --help | grep x86-64-v3" --out=json 2>/dev/null) || true
+
+  while IFS= read -r node; do
+    [[ -z "$node" ]] && continue
+    result=$(jq -r --arg node "$node" '.[$node] // empty' <<< "$cpu_results" 2>/dev/null)
+    if [[ -z "$result" || "$result" == *"did not return"* ]]; then
+      offline+=("$node")
+    elif [[ "$result" != *"x86-64-v3 (supported"* ]]; then
+      # glibc appends "(supported, searched)" only when supported; the open paren keeps
+      # this from matching a future "(unsupported".
+      unsupported+=("$node")
+    fi
+  done <<< "$expected_nodes"
+
+  if [[ ${#unsupported[@]} -eq 0 && ${#offline[@]} -eq 0 ]]; then
+    printf "\nAll Elastic Agent nodes support x86-64-v3. We can proceed with SOUP.\n\n"
+    return
+  fi
+
+  echo ""
+  if [[ ${#unsupported[@]} -gt 0 ]]; then
+    echo "The following node(s) do NOT support the x86-64-v3 CPU instruction set:"
+    printf '  %s\n' "${unsupported[@]}"
+    echo ""
+    echo "Upstream Elastic now builds its binaries for x86-64-v3, so these nodes can no"
+    echo "longer run Elastic. Upgrading them WILL BREAK them."
+    echo ""
+  fi
+  if [[ ${#offline[@]} -gt 0 ]]; then
+    echo "The following node(s) did not respond and could not be checked:"
+    printf '  %s\n' "${offline[@]}"
+    echo ""
+    echo "These nodes are offline, so we cannot confirm they support x86-64-v3, which"
+    echo "upstream Elastic now requires."
+    echo ""
+  fi
+
+  if [[ -n $UNATTENDED ]]; then
+    echo "Unattended mode cannot prompt for an override. Re-run soup interactively, or pass --skip-cpu-check to bypass this check."
+    no_soup_for_you
+  fi
+
+  read -rp "Type 'override' to continue anyway, or press Enter to exit: " confirm
+  if [[ "${confirm,,}" == "override" ]]; then
+    printf "\nOverride accepted. Continuing at your own risk.\n\n"
+  else
+    no_soup_for_you
+  fi
+}
+
 check_fleet_server() {
   echo "Checking that Elastic Fleet Server is responding."
   # Modeled on the wait_for_so-elastic-fleet state check in elasticfleet/enabled.sls,
@@ -2088,6 +2168,8 @@ main() {
   echo "Let's see if we need to update Security Onion."
   upgrade_check
 
+  check_cpu_compatibility
+
   upgrade_space
 
   echo "Verifying Elasticsearch version compatibility across the grid before upgrading."
@@ -2366,6 +2448,17 @@ fi
   echo "### soup has been served at $(date) ###"
 }
 
+SKIP_CPU_CHECK=false
+declare -a SOUP_ARGS=()
+for arg in "$@"; do
+  if [[ "$arg" == "--skip-cpu-check" ]]; then
+    SKIP_CPU_CHECK=true
+  else
+    SOUP_ARGS+=("$arg")
+  fi
+done
+set -- "${SOUP_ARGS[@]}"
+
 while getopts ":b:f:y" opt; do
   case ${opt} in
     b )
@@ -2389,7 +2482,7 @@ while getopts ":b:f:y" opt; do
       ISOLOC="$OPTARG"
     ;;
     \? )
-      echo "Usage: soup [-b] [-y] [-f <iso location>]"
+      echo "Usage: soup [-b] [-y] [-f <iso location>] [--skip-cpu-check]"
       exit 1
     ;;
     : )

setup/so-functions

@@ -976,50 +976,27 @@ docker_seed_update() {
 	set_progress_str "$docker_seed_update_percent" "Downloading $name"
 }
 
-registry_tar_has_soversion() {
-	local tarfile=$1
-	local version=$2
-
-	[ -n "$version" ] || return 1
-	[ -f "$tarfile" ] || return 1
-	# grep -q in a pipe SIGPIPEs tar; under pipefail a match would look like failure.
-	grep -qF "/_manifests/tags/${version}/" < <(tar -tf "$tarfile" 2>/dev/null)
-}
-
 docker_seed_registry() {
 	local VERSION="$SOVERSION"
-	local REGISTRY_TAR=/nsm/docker-registry/docker/registry.tar
 
-	if [ -f "$REGISTRY_TAR" ] && registry_tar_has_soversion "$REGISTRY_TAR" "$VERSION"; then
+	if [ -f /nsm/docker-registry/docker/registry.tar ]; then
-		info "Extracting $REGISTRY_TAR"
+		logCmd "tar xvf /nsm/docker-registry/docker/registry.tar -C /nsm/docker-registry/docker"
-		if ! tar xvf "$REGISTRY_TAR" -C /nsm/docker-registry/docker >> "$setup_log" 2>&1; then
+		logCmd "rm /nsm/docker-registry/docker/registry.tar"
-			error "Failed to extract $REGISTRY_TAR"
-			return 1
-		fi
-		logCmd "rm -f $REGISTRY_TAR"
-		return 0
-	elif [ -f "$REGISTRY_TAR" ]; then
-		info "Not using $REGISTRY_TAR; it has no images tagged '${VERSION}'"
-		if [[ "$is_airgap" == true ]]; then
-			error "Airgap install: $REGISTRY_TAR does not match Security Onion version '${VERSION}'. Use the ISO for this version."
-			return 1
-		fi
 	elif [[ -d /nsm/docker-registry/docker/registry && ( -f /etc/SOCLOUD || "$is_airgap" == true ) ]]; then
 		echo "Using existing docker registry content"
-		return 0
-	fi
-
-	if [ "$install_type" == 'IMPORT' ]; then
-		container_list 'so-import'	
 	else
-		container_list
+		if [ "$install_type" == 'IMPORT' ]; then
-	fi
+			container_list 'so-import'	
+		else
+			container_list
+		fi
 
-	docker_seed_update_percent=25
+		docker_seed_update_percent=25
 
-	update_docker_containers 'netinstall' '' 'docker_seed_update' '/dev/stdout' 2>&1 | tee -a "$setup_log"
+		update_docker_containers 'netinstall' '' 'docker_seed_update' '/dev/stdout' 2>&1 | tee -a "$setup_log"
         # Use pipe exit status of 'update_docker_containers' for return code
-	return ${PIPESTATUS[0]}
+		return ${PIPESTATUS[0]}
+	fi
 }
 
 elasticfleet_pillar() {