dedup artifact registry config & split ea artifacts out of /opt/socore/html/ into dedicated /opt/socore/elastic-artifacts/artifacts/

surface ilm policy errors in so-elasticsearch-ilm-policy-load output

salt/desktop/packages.sls

@@ -438,6 +438,10 @@ install_networkminer:
   pkg.latest:
     - name: securityonion-networkminer
 
+install_so_crates:
+  pkg.latest:
+    - name: securityonion-so-crates
+
 {% else %}
 
 desktop_packages_os_fail:

salt/elasticsearch/cluster.sls

@@ -107,6 +107,12 @@ so-elasticsearch-system-indices-patch:
       - file: so-elasticsearch-system-indices-patch-script
 {%  endif %}
 
+so-elasticsearch-ilm-policy-reload:
+  file.absent:
+    - name: /opt/so/state/esilm.txt
+    - onchanges:
+      - file: so-elasticsearch-ilm-policy-load-script
+
 # heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)
 so-elasticsearch-ilm-policy-load:
   cmd.run:
@@ -115,8 +121,9 @@ so-elasticsearch-ilm-policy-load:
     - require:
       - docker_container: so-elasticsearch
       - file: so-elasticsearch-ilm-policy-load-script
-    - onchanges:
-      - file: so-elasticsearch-ilm-policy-load-script
+      - http: wait_for_so-elasticsearch
+      - file: so-elasticsearch-ilm-policy-reload
+    - unless: test -f /opt/so/state/esilm.txt
 
 so-elasticsearch-templates-reload:
   file.absent:

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load

@@ -7,11 +7,17 @@
 . /usr/sbin/so-common
 
 MAX_JOBS=${MAX_ILM_JOBS:-10}
+ILM_STATEFILE=/opt/so/state/esilm.txt
+rm -f "$ILM_STATEFILE"
 
+check_elasticsearch_responsive
+
+ILM_TMP_DIR=$(mktemp -d)
+trap 'rm -rf "$ILM_TMP_DIR"' EXIT
 # Lock used to serialize block writes so concurrent jobs never interleave their output.
-ILM_OUTPUT_LOCK=$(mktemp)
-ILM_FAIL_FILE=$(mktemp)
-trap 'rm -f "$ILM_OUTPUT_LOCK" "$ILM_FAIL_FILE"' EXIT
+ILM_OUTPUT_LOCK="$ILM_TMP_DIR/output.lock"
+ILM_FAIL_FILE="$ILM_TMP_DIR/failed"
+touch "$ILM_FAIL_FILE"
 
 # Policies are loaded concurrently (up to MAX_JOBS at a time) for speed. Each policy's block is
 # printed the moment its curl returns, so output appears in COMPLETION ORDER, not the order
@@ -20,13 +26,30 @@ echo "Loading ILM policies concurrently; output below appears in completion orde
 echo
 
 put_policy() {
-  local desc="$1" policyname="$2" data="$3" result rc=0
-  if ! result=$(curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L --fail \
-    -X PUT "https://localhost:9200/_ilm/policy/${policyname}" \
-    -H 'Content-Type: application/json' -d"${data}" 2>&1); then
-    rc=1
-  elif ! jq -e '.acknowledged == true' <<<"$result" >/dev/null 2>&1; then
-    rc=1
+  local desc="$1" policyname="$2" data="$3" rc=1 attempt out http_code body result reason tmpdata
+  tmpdata=$(mktemp "$ILM_TMP_DIR/${policyname}.XXXXXX.json")
+  printf '%s' "$data" >"$tmpdata"
+
+  for attempt in 1 2 3; do
+    out=$(so-elasticsearch-query "_ilm/policy/${policyname}" -d@"${tmpdata}" -XPUT -w '\n%{http_code}' 2>/dev/null)
+    http_code=${out##*$'\n'} body=${out%$'\n'*}
+
+    if [[ "$http_code" == 200 ]] && jq -e '.acknowledged == true' <<<"$body" >/dev/null 2>&1; then
+      rc=0
+      break
+    fi
+    # 429 = ES rejected/throttled (may succeed later); other 4xx will not
+    [[ "$http_code" == 4?? && "$http_code" != 429 ]] && break
+    (( attempt < 3 )) && sleep 3
+  done
+
+  if (( rc == 0 )); then
+    result='{"acknowledged":true}'
+  else
+    result=${body:-"HTTP ${http_code:-000}"}
+    # flatten tabs/newlines so the TAB fail-file stays one record
+    reason=$(jq -r '(if (.error|type) == "object" then (.error.reason // .error.type) elif (.error|type) == "string" then .error else empty end) // empty | tostring | gsub("[\t\r\n]+"; " ")' <<<"$body" 2>/dev/null)
+    reason="HTTP ${http_code:-000}${reason:+: ${reason}}"
   fi
 
   # curl above ran in parallel; serialize just this block write so concurrent jobs never interleave.
@@ -34,7 +57,7 @@ put_policy() {
     flock 200
     printf 'Setting up %s policy...\n%s\n\n' "${desc}" "${result}"
     if (( rc != 0 )); then
-      printf '%s\n' "${policyname}" >>"$ILM_FAIL_FILE"
+      printf '%s\t%s\n' "${policyname}" "${reason}" >>"$ILM_FAIL_FILE"
     fi
   } 200>>"${ILM_OUTPUT_LOCK}"
 
@@ -82,10 +105,15 @@ wait || true
 
 if [[ -s "$ILM_FAIL_FILE" ]]; then
   echo "ERROR: Failed to load ILM policy(s):"
-  while read -r POLICY; do
-    echo " - $POLICY"
+  while IFS=$'\t' read -r POLICY REASON; do
+    if [[ -n "$REASON" ]]; then
+      echo " - ${POLICY}: ${REASON}"
+    else
+      echo " - ${POLICY}"
+    fi
   done < "$ILM_FAIL_FILE"
   exit 1
 else
   echo "Successfully loaded all ILM policies."
+  touch "$ILM_STATEFILE"
 fi

salt/nginx/enabled.sls

@@ -54,9 +54,9 @@ so-nginx:
       - /opt/so/log/nginx/:/var/log/nginx:rw
       - /opt/so/tmp/nginx/:/var/lib/nginx:rw
       - /opt/so/tmp/nginx/:/run:rw
-      - /nsm/elastic-fleet/so_agent-installers/:/opt/socore/html/packages
-      - /nsm/elastic-fleet/artifacts/:/opt/socore/html/artifacts 
+      - /nsm/elastic-fleet/artifacts/:/opt/socore/elastic-artifacts/artifacts:ro
       {% if GLOBALS.is_manager %}
+      - /nsm/elastic-fleet/so_agent-installers/:/opt/socore/html/packages
       - /etc/pki/managerssl.crt:/etc/pki/nginx/server.crt:ro
       - /etc/pki/managerssl.key:/etc/pki/nginx/server.key:ro
       # ATT&CK Navigator binds

salt/nginx/etc/nginx.conf

@@ -4,6 +4,8 @@
 {%- set role = grains.id.split('_') | last %}
 {%- set influxpass = salt['pillar.get']('secrets:influx_pass') %}
 {%- set influxauth = ('so:' + influxpass) | base64_encode %}
+{%- set all_names = [GLOBALS.hostname, GLOBALS.url_base] + NGINXMERGED.ssl.alt_names %}
+{%- set full_server_name = all_names | unique | join(' ') %}
 
 worker_processes auto;
 error_log /var/log/nginx/error.log;
@@ -39,52 +41,31 @@ http {
 
 	include /etc/nginx/conf.d/*.conf;
 
-	{%- if role in ['fleet'] %}
-
-	server {
-		listen 8443;
-		server_name {{ GLOBALS.hostname }};
-		root /opt/socore/html;
-		location /artifacts/ {
-			try_files $uri =206;
-			proxy_read_timeout    90;
-			proxy_connect_timeout 90;
-			proxy_set_header      Host $host;
-			proxy_set_header      X-Real-IP $remote_addr;
-			proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
-			proxy_set_header      Proxy "";
-			proxy_set_header      X-Forwarded-Proto $scheme;
-		}
-	}
-
-    {%- endif %}
-
 	{%- if GLOBALS.is_manager %}
-	{%- set all_names = [GLOBALS.hostname, GLOBALS.url_base] + NGINXMERGED.ssl.alt_names %}
-	{%- set full_server_name = all_names | unique | join(' ') %}
-
 	server {
 		listen 80 default_server;
 		server_name _;
 		return 307 https://{{ GLOBALS.url_base }}$request_uri;
 	}
-
+  {%- endif %}
+  {%- if GLOBALS.is_manager or role == 'fleet' %}
 	server {
 		listen 8443;
+    {%- if GLOBALS.is_manager %}
 		server_name {{ full_server_name }};
-		root /opt/socore/html;
+    {%- else %}
+    server_name {{ GLOBALS.hostname }};
+    {%- endif %}
+		root /opt/socore/elastic-artifacts;
 		location /artifacts/ {
-			try_files $uri =206;
-			proxy_read_timeout    90;
-			proxy_connect_timeout 90;
-			proxy_set_header      Host $host;
-			proxy_set_header      X-Real-IP $remote_addr;
-			proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
-			proxy_set_header      Proxy "";
-			proxy_set_header      X-Forwarded-Proto $scheme;
+			try_files $uri =404;
+		}
+		location / {
+			return 404;
 		}
 	}
-
+  {%- endif %}
+  {%- if GLOBALS.is_manager %}
 	server {
 		listen 443 ssl default_server;
 		http2 on;

setup/so-functions

@@ -976,27 +976,50 @@ docker_seed_update() {
 	set_progress_str "$docker_seed_update_percent" "Downloading $name"
 }
 
+registry_tar_has_soversion() {
+	local tarfile=$1
+	local version=$2
+
+	[ -n "$version" ] || return 1
+	[ -f "$tarfile" ] || return 1
+	# grep -q in a pipe SIGPIPEs tar; under pipefail a match would look like failure.
+	grep -qF "/_manifests/tags/${version}/" < <(tar -tf "$tarfile" 2>/dev/null)
+}
+
 docker_seed_registry() {
 	local VERSION="$SOVERSION"
+	local REGISTRY_TAR=/nsm/docker-registry/docker/registry.tar
 
-	if [ -f /nsm/docker-registry/docker/registry.tar ]; then
-		logCmd "tar xvf /nsm/docker-registry/docker/registry.tar -C /nsm/docker-registry/docker"
-		logCmd "rm /nsm/docker-registry/docker/registry.tar"
+	if [ -f "$REGISTRY_TAR" ] && registry_tar_has_soversion "$REGISTRY_TAR" "$VERSION"; then
+		info "Extracting $REGISTRY_TAR"
+		if ! tar xvf "$REGISTRY_TAR" -C /nsm/docker-registry/docker >> "$setup_log" 2>&1; then
+			error "Failed to extract $REGISTRY_TAR"
+			return 1
+		fi
+		logCmd "rm -f $REGISTRY_TAR"
+		return 0
+	elif [ -f "$REGISTRY_TAR" ]; then
+		info "Not using $REGISTRY_TAR; it has no images tagged '${VERSION}'"
+		if [[ "$is_airgap" == true ]]; then
+			error "Airgap install: $REGISTRY_TAR does not match Security Onion version '${VERSION}'. Use the ISO for this version."
+			return 1
+		fi
 	elif [[ -d /nsm/docker-registry/docker/registry && ( -f /etc/SOCLOUD || "$is_airgap" == true ) ]]; then
 		echo "Using existing docker registry content"
-	else
-		if [ "$install_type" == 'IMPORT' ]; then
-			container_list 'so-import'	
-		else
-			container_list
-		fi
-
-		docker_seed_update_percent=25
-
-		update_docker_containers 'netinstall' '' 'docker_seed_update' '/dev/stdout' 2>&1 | tee -a "$setup_log"
-        # Use pipe exit status of 'update_docker_containers' for return code
-		return ${PIPESTATUS[0]}
+		return 0
 	fi
+
+	if [ "$install_type" == 'IMPORT' ]; then
+		container_list 'so-import'	
+	else
+		container_list
+	fi
+
+	docker_seed_update_percent=25
+
+	update_docker_containers 'netinstall' '' 'docker_seed_update' '/dev/stdout' 2>&1 | tee -a "$setup_log"
+        # Use pipe exit status of 'update_docker_containers' for return code
+	return ${PIPESTATUS[0]}
 }
 
 elasticfleet_pillar() {