Compare commits

..
Author SHA1 Message Date
reyesj2 13eaea3656 Stop unused Fleet latest transforms that hold empty shards
After addon package install, stop transforms for packages that are not
on an enabled agent policy. On soup/highstate, stop remaining latest
transforms whose source and destination are empty and whose package is
not assigned to a policy. An operator start in Kibana is left running.
2026-11-04 15:35:33 -06:00
14 changed files with 406 additions and 120 deletions

No files matched your search

-1
View File
@@ -14,7 +14,6 @@ body:
- 3.2.0
- 3.3.0
- 3.4.0
- 3.5.0
- Other (please provide detail below)
validations:
required: true
+11 -11
View File
@@ -1,17 +1,17 @@
### 3.4.0-20261102 ISO image released on 2026/11/02
### 3.3.0-20260911 ISO image released on 2026/09/11
### Download and Verify
3.4.0-20261102 ISO image:
https://download.securityonion.net/file/securityonion/securityonion-3.4.0-20261102.iso
3.3.0-20260911 ISO image:
https://download.securityonion.net/file/securityonion/securityonion-3.3.0-20260911.iso
MD5: FB0380AE996E175D6CC7E7A35D8CC178
SHA1: D8960F1DA352710BCA209773BF0B3AE8791D0D1B
SHA256: DFC4C87668836852A328EC7A0B755C8C1511FF0E43D423DEE23047A95F676D99
MD5: 12B18433D3A2198A185892FF79CF638F
SHA1: 2B3C2E1FA7A78ED1F956E7EDCC12E32593C14EEE
SHA256: 0938C73B76CE30EC9E4394D312C79EA7CAC721B6818541697279A6221F7D870D
Signature for ISO image:
https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.4.0-20261102.iso.sig
https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.3.0-20260911.iso.sig
Signing key:
https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/main/KEYS
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/
Download the signature file for the ISO:
```
wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.4.0-20261102.iso.sig
wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.3.0-20260911.iso.sig
```
Download the ISO image:
```
wget https://download.securityonion.net/file/securityonion/securityonion-3.4.0-20261102.iso
wget https://download.securityonion.net/file/securityonion/securityonion-3.3.0-20260911.iso
```
Verify the downloaded ISO image using the signature file:
```
gpg --verify securityonion-3.4.0-20261102.iso.sig securityonion-3.4.0-20261102.iso
gpg --verify securityonion-3.3.0-20260911.iso.sig securityonion-3.3.0-20260911.iso
```
The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
```
gpg: Signature made Sat 31 Oct 2026 09:27:50 AM EDT using RSA key ID FE507013
gpg: Signature made Fri 11 Sep 2026 11:23:56 AM EDT using RSA key ID FE507013
gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
gpg: WARNING: This key is not certified with a trusted signature!
gpg: There is no indication that the signature belongs to the owner.
+1 -1
View File
@@ -1 +1 @@
3.5.0
3.4.0
-4
View File
@@ -438,10 +438,6 @@ install_networkminer:
pkg.latest:
- name: securityonion-networkminer
install_so_crates:
pkg.latest:
- name: securityonion-so-crates
{% else %}
desktop_packages_os_fail:
+11
View File
@@ -112,6 +112,17 @@ so-elastic-fleet-addon-integrations:
- http: wait_for_so-kibana
- cmd: so-elasticsearch-templates-after-fleet-packages
so-elastic-fleet-transforms-manage:
cmd.run:
- name: /usr/sbin/so-elastic-fleet-transforms-manage
- require:
- http: wait_for_so-kibana
- file: elasticfleet_sbin
- cmd: so-elastic-fleet-package-upgrade
- cmd: so-elastic-fleet-integration-upgrade
- cmd: so-elastic-fleet-integrations
- cmd: so-elastic-fleet-addon-integrations
{% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %}
so-elastic-defend-manage-filters-file-watch:
cmd.run:
@@ -0,0 +1,225 @@
#!/bin/bash
#
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
#
# 1. List transforms (source/dest come from the transform config).
# 2. List enabled Fleet package policies.
# 3. Unused packages: count only those source/dest indices, then stop and
# delete empty dests. --after-install skips the counts and stops unused.
. /usr/sbin/so-common
. /usr/sbin/so-elastic-fleet-common
PIVOT_PREFIX='logs-elasticsearch.index_pivot-default-'
STATE_FILE=/opt/so/state/esfleet_stopped_transforms.json
DRY_RUN=false
AFTER_INSTALL=false
usage() {
echo "Usage: $0 [--dry-run] [--after-install]" >&2
exit 1
}
while [[ $# -gt 0 ]]; do
case "$1" in
--dry-run) DRY_RUN=true ;;
--after-install) AFTER_INSTALL=true ;;
-h|--help) usage ;;
*) usage ;;
esac
shift
done
es() {
so-elasticsearch-query "$@" --fail --retry 2 --retry-delay 3
}
skip() {
echo "Warning: $1; skipping transform cleanup." >&2
exit 0
}
run() {
local desc=$1
shift
if [[ "$DRY_RUN" == true ]]; then
echo "[dry-run] $desc"
return 0
fi
echo "$desc"
es "$@" >/dev/null
}
uri() {
jq -rn --arg id "$1" '$id|@uri'
}
in_use_packages() {
local page=1 collected=0 count total resp packages='[]'
while true; do
resp=$(fleet_api "package_policies?perPage=100&page=$page") || return 1
jq -e '(.items | type == "array") and (.total | type == "number")' <<<"$resp" >/dev/null || return 1
packages=$(jq -c --argjson acc "$packages" '
$acc + [.items[]
| select(.enabled == true)
| select(([.policy_ids[]?, .policy_id] | map(select(. != null and . != "")) | length) > 0)
| .package.name
| select(. != null and . != "")]' <<<"$resp")
count=$(jq '.items | length' <<<"$resp")
total=$(jq '.total' <<<"$resp")
collected=$((collected + count))
[[ "$count" -eq 0 || "$collected" -ge "$total" ]] && break
page=$((page + 1))
[[ "$page" -gt 100 ]] && return 1
done
[[ "$collected" -eq "$total" ]] || return 1
jq -c 'unique' <<<"$packages"
}
# Page into files: a full transform list exceeds ARG_MAX.
page_es() {
local path=$1 outfile=$2 from=0 size=1000 count got page_len
echo '[]' > "$outfile"
while true; do
es "${path}?from=${from}&size=${size}" > "$WORKDIR/page.json" || return 1
jq -e '(.count | type == "number") and (.transforms | type == "array")' "$WORKDIR/page.json" >/dev/null || return 1
count=$(jq -r '.count' "$WORKDIR/page.json")
jq -c --slurpfile page "$WORKDIR/page.json" '. + ($page[0].transforms // [])' "$outfile" > "$WORKDIR/next.json" || return 1
mv "$WORKDIR/next.json" "$outfile"
got=$(jq 'length' "$outfile")
page_len=$(jq '.transforms | length' "$WORKDIR/page.json")
[[ "$page_len" -eq 0 || "$got" -ge "$count" ]] && break
from=$((from + size))
[[ "$from" -gt 100000 ]] && return 1
done
[[ "$got" -eq "$count" ]]
}
load_transforms() {
page_es "_transform" "$WORKDIR/tf-configs.json" || return 1
page_es "_transform/_stats" "$WORKDIR/tf-stats.json" || return 1
jq -c --slurpfile stats "$WORKDIR/tf-stats.json" --arg pivot "$PIVOT_PREFIX" '
($stats[0] | map({key: .id, value: .state}) | from_entries) as $states
| [.[] | {
id,
package: (._meta.package.name // null),
state: ($states[.id] // "unknown"),
dest: (.dest.index // null),
create_time: (.create_time // null),
source: (.source.index | if type == "array" then . elif type == "string" then [.] else [] end),
pivot: (.id | startswith($pivot)),
latest: ((.latest != null) or ((.dest.index // "") | contains("_latest")))
}
| select(.pivot or .latest)
]' "$WORKDIR/tf-configs.json" > "$WORKDIR/transforms.json"
}
doc_count() {
local pattern=$1 count
count=$(es "${pattern}/_count?ignore_unavailable=true" | jq -er '.count') || return 1
[[ "$count" =~ ^[0-9]+$ ]] || return 1
echo "$count"
}
status "Elastic Fleet transform cleanup (after_install=$AFTER_INSTALL dry_run=$DRY_RUN)"
WORKDIR=$(mktemp -d)
trap 'rm -rf "$WORKDIR"' EXIT
load_transforms || skip "Elasticsearch transforms could not be retrieved in full"
IN_USE=$(in_use_packages) || skip "Fleet package policies could not be retrieved in full"
jq -e 'length > 0' <<<"$IN_USE" >/dev/null || skip "no enabled Fleet package policies are assigned to an agent policy"
STATE=$(jq -c '{stopped: ((.stopped | objects) // {}), user_started: ((.user_started | arrays) // [])}' "$STATE_FILE" 2>/dev/null)
jq -e '.stopped and .user_started' <<<"$STATE" >/dev/null 2>&1 || STATE='{"stopped":{},"user_started":[]}'
# Same create_time but running again: operator started it after we stopped it.
STATE=$(jq -c --slurpfile t "$WORKDIR/transforms.json" '
($t[0] | map({key: .id, value: .}) | from_entries) as $cur
| (.stopped | to_entries | map(select($cur[.key] != null and $cur[.key].create_time == .value))) as $tracked
| {
stopped: ($tracked | map(select($cur[.key].state == "stopped")) | from_entries),
user_started: ((.user_started + [$tracked[] | select($cur[.key].state | IN("started", "indexing")) | .key])
| unique | map(select($cur[.] != null and ($cur[.].state | IN("stopped", "failed") | not))))
}' <<<"$STATE")
# Unused = latest/pivot with a package, not on a policy, not operator-started.
# Pivot is always unused (see so-elastic-fleet-package-upgrade).
jq -c --argjson inuse "$IN_USE" --argjson state "$STATE" '
def unused:
if .pivot then true
elif .package == null then false
elif (.id | IN($state.user_started[]?)) then false
elif (.package | IN($inuse[]?)) then false
else true
end;
{
unused: [.[] | select(unused)],
start: [.[] | select(.package != null and (.state | IN("stopped", "failed"))
and ($state.stopped[.id] != null) and (.pivot | not)
and (.package | IN($inuse[]?)))]
}' "$WORKDIR/transforms.json" > "$WORKDIR/split.json"
jq -c '.unused' "$WORKDIR/split.json" > "$WORKDIR/unused.json"
if [[ "$AFTER_INSTALL" == true ]]; then
jq -c '[.[] | select(.state | IN("started", "indexing", "failed"))]' "$WORKDIR/unused.json" > "$WORKDIR/stop.json"
else
echo '{}' > "$WORKDIR/counts.json"
while IFS= read -r pattern; do
[[ -n "$pattern" ]] || continue
n=$(doc_count "$pattern") || skip "could not count documents in $pattern"
jq -c --arg p "$pattern" --argjson n "$n" '. + {($p): $n}' "$WORKDIR/counts.json" > "$WORKDIR/counts-next.json"
mv "$WORKDIR/counts-next.json" "$WORKDIR/counts.json"
done < <(jq -r '[.[].dest | select(. != null and . != "")] + [.[].source[]?] | unique | .[]' "$WORKDIR/unused.json")
jq -c --slurpfile c "$WORKDIR/counts.json" '
($c[0] // {}) as $n
| [.[] | select(.state | IN("started", "indexing", "failed"))
| select((if .dest == null or .dest == "" then 0 else ($n[.dest] // 0) end) == 0)
| select(([.source[]? | $n[.] // 0] | add // 0) == 0)]' \
"$WORKDIR/unused.json" > "$WORKDIR/stop.json"
fi
echo "in_use=$(jq -r 'join(",")' <<<"$IN_USE") unused=$(jq 'length' "$WORKDIR/unused.json") stop=$(jq 'length' "$WORKDIR/stop.json") start=$(jq '.start | length' "$WORKDIR/split.json") dry_run=$DRY_RUN"
while IFS= read -r id; do
[[ -n "$id" ]] || continue
force=
jq -e --arg id "$id" '.[] | select(.id == $id and .state == "failed")' "$WORKDIR/stop.json" >/dev/null && force='&force=true'
if run "stop transform $id" "_transform/$(uri "$id")/_stop?wait_for_completion=true&timeout=60s${force}" -XPOST; then
[[ "$DRY_RUN" == true ]] && continue
ct=$(jq -c --arg id "$id" '.[] | select(.id == $id) | .create_time' "$WORKDIR/stop.json")
STATE=$(jq -c --arg id "$id" --argjson ct "$ct" '.stopped[$id] = $ct | .user_started -= [$id]' <<<"$STATE")
fi
done < <(jq -r '.[].id' "$WORKDIR/stop.json")
while IFS= read -r id; do
[[ -n "$id" ]] || continue
if run "start transform $id" "_transform/$(uri "$id")/_start" -XPOST; then
[[ "$DRY_RUN" == false ]] && STATE=$(jq -c --arg id "$id" 'del(.stopped[$id])' <<<"$STATE")
fi
done < <(jq -r '.start[].id' "$WORKDIR/split.json")
if [[ "$DRY_RUN" == false ]]; then
mkdir -p "$(dirname "$STATE_FILE")"
tmp=$(mktemp "${STATE_FILE}.XXXXXX") && printf '%s\n' "$STATE" > "$tmp" && mv -f "$tmp" "$STATE_FILE"
fi
DELETED=0
while IFS= read -r dest; do
[[ -n "$dest" ]] || continue
if [[ "$DRY_RUN" == true ]]; then
echo "[dry-run] delete empty transform destination $dest"
DELETED=$((DELETED + 1))
continue
fi
n=$(doc_count "$dest") || continue
[[ "$n" -eq 0 ]] || continue
run "delete empty transform destination $dest" "$(uri "$dest")" -XDELETE && DELETED=$((DELETED + 1))
done < <(jq -r '.[].dest | select(. != null and . != "")' "$WORKDIR/stop.json" | sort -u)
echo "stopped=$(jq -r '.stopped | length' <<<"$STATE") user_started=$(jq -r '.user_started | length' <<<"$STATE") dests_removed=$DELETED"
@@ -137,6 +137,7 @@ def needs($sub;$autoup;$excluded;$inuse):
done
# cleanup any temp files for chunked package install
rm -f ${pkg_filename}_*.json $BULK_INSTALL_PACKAGE_LIST
/usr/sbin/so-elastic-fleet-transforms-manage --after-install || echo "Warning: transform cleanup after package install failed; highstate will retry."
else
echo "Elastic integrations don't appear to need installation/updating..."
fi
+2 -9
View File
@@ -107,12 +107,6 @@ so-elasticsearch-system-indices-patch:
- file: so-elasticsearch-system-indices-patch-script
{% endif %}
so-elasticsearch-ilm-policy-reload:
file.absent:
- name: /opt/so/state/esilm.txt
- onchanges:
- file: so-elasticsearch-ilm-policy-load-script
# heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)
so-elasticsearch-ilm-policy-load:
cmd.run:
@@ -121,9 +115,8 @@ so-elasticsearch-ilm-policy-load:
- require:
- docker_container: so-elasticsearch
- file: so-elasticsearch-ilm-policy-load-script
- http: wait_for_so-elasticsearch
- file: so-elasticsearch-ilm-policy-reload
- unless: test -f /opt/so/state/esilm.txt
- onchanges:
- file: so-elasticsearch-ilm-policy-load-script
so-elasticsearch-templates-reload:
file.absent:
@@ -7,17 +7,11 @@
. /usr/sbin/so-common
MAX_JOBS=${MAX_ILM_JOBS:-10}
ILM_STATEFILE=/opt/so/state/esilm.txt
rm -f "$ILM_STATEFILE"
check_elasticsearch_responsive
ILM_TMP_DIR=$(mktemp -d)
trap 'rm -rf "$ILM_TMP_DIR"' EXIT
# Lock used to serialize block writes so concurrent jobs never interleave their output.
ILM_OUTPUT_LOCK="$ILM_TMP_DIR/output.lock"
ILM_FAIL_FILE="$ILM_TMP_DIR/failed"
touch "$ILM_FAIL_FILE"
ILM_OUTPUT_LOCK=$(mktemp)
ILM_FAIL_FILE=$(mktemp)
trap 'rm -f "$ILM_OUTPUT_LOCK" "$ILM_FAIL_FILE"' EXIT
# Policies are loaded concurrently (up to MAX_JOBS at a time) for speed. Each policy's block is
# printed the moment its curl returns, so output appears in COMPLETION ORDER, not the order
@@ -26,30 +20,13 @@ echo "Loading ILM policies concurrently; output below appears in completion orde
echo
put_policy() {
local desc="$1" policyname="$2" data="$3" rc=1 attempt out http_code body result reason tmpdata
tmpdata=$(mktemp "$ILM_TMP_DIR/${policyname}.XXXXXX.json")
printf '%s' "$data" >"$tmpdata"
for attempt in 1 2 3; do
out=$(so-elasticsearch-query "_ilm/policy/${policyname}" -d@"${tmpdata}" -XPUT -w '\n%{http_code}' 2>/dev/null)
http_code=${out##*$'\n'} body=${out%$'\n'*}
if [[ "$http_code" == 200 ]] && jq -e '.acknowledged == true' <<<"$body" >/dev/null 2>&1; then
rc=0
break
fi
# 429 = ES rejected/throttled (may succeed later); other 4xx will not
[[ "$http_code" == 4?? && "$http_code" != 429 ]] && break
(( attempt < 3 )) && sleep 3
done
if (( rc == 0 )); then
result='{"acknowledged":true}'
else
result=${body:-"HTTP ${http_code:-000}"}
# flatten tabs/newlines so the TAB fail-file stays one record
reason=$(jq -r '(if (.error|type) == "object" then (.error.reason // .error.type) elif (.error|type) == "string" then .error else empty end) // empty | tostring | gsub("[\t\r\n]+"; " ")' <<<"$body" 2>/dev/null)
reason="HTTP ${http_code:-000}${reason:+: ${reason}}"
local desc="$1" policyname="$2" data="$3" result rc=0
if ! result=$(curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L --fail \
-X PUT "https://localhost:9200/_ilm/policy/${policyname}" \
-H 'Content-Type: application/json' -d"${data}" 2>&1); then
rc=1
elif ! jq -e '.acknowledged == true' <<<"$result" >/dev/null 2>&1; then
rc=1
fi
# curl above ran in parallel; serialize just this block write so concurrent jobs never interleave.
@@ -57,7 +34,7 @@ put_policy() {
flock 200
printf 'Setting up %s policy...\n%s\n\n' "${desc}" "${result}"
if (( rc != 0 )); then
printf '%s\t%s\n' "${policyname}" "${reason}" >>"$ILM_FAIL_FILE"
printf '%s\n' "${policyname}" >>"$ILM_FAIL_FILE"
fi
} 200>>"${ILM_OUTPUT_LOCK}"
@@ -105,15 +82,10 @@ wait || true
if [[ -s "$ILM_FAIL_FILE" ]]; then
echo "ERROR: Failed to load ILM policy(s):"
while IFS=$'\t' read -r POLICY REASON; do
if [[ -n "$REASON" ]]; then
echo " - ${POLICY}: ${REASON}"
else
echo " - ${POLICY}"
fi
while read -r POLICY; do
echo " - $POLICY"
done < "$ILM_FAIL_FILE"
exit 1
else
echo "Successfully loaded all ILM policies."
touch "$ILM_STATEFILE"
fi
+94 -1
View File
@@ -121,6 +121,9 @@ check_err() {
161)
echo 'Required intermediate Elasticsearch upgrade not complete'
;;
162)
echo 'One or more Elastic Agent nodes do not support the x86-64-v3 CPU instruction set'
;;
170)
echo "Intermediate upgrade completed successfully to $next_step_so_version, but next soup to Security Onion $originally_requested_so_version could not be started automatically."
echo "Start soup again manually to continue the upgrade to Security Onion $originally_requested_so_version."
@@ -348,6 +351,83 @@ check_cluster_health() {
exit 0
}
no_soup_for_you() {
echo ""
echo "No soup for you!"
exit 162
}
check_cpu_compatibility() {
# Roles running a container built from the so-elastic-agent image; mirrors the
# elasticagent and elasticfleet entries in salt/reactor/pillar_push_map.yaml.
local cpu_target='G@role:so-heavynode or G@role:so-eval or G@role:so-fleet or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone'
local expected_nodes cpu_results node result confirm
local -a unsupported=() offline=()
echo "Checking that Elastic Agent nodes support the x86-64-v3 CPU instruction set now required by Elastic."
if [[ "$SKIP_CPU_CHECK" == "true" ]]; then
printf "\nSkipping the x86-64-v3 CPU check because --skip-cpu-check was specified.\n\n"
return
fi
# Nodes that never answer are absent from the results, so diff against who should have.
expected_nodes=$(salt -C "$cpu_target" --preview-target --out=json 2>/dev/null | jq -r '.[]?') || true
if [[ -z "$expected_nodes" ]]; then
printf "\nCould not determine which nodes run the Elastic Agent, so the x86-64-v3 CPU check cannot run.\n"
no_soup_for_you
fi
cpu_results=$(salt -t 30 -C "$cpu_target" cmd.run "/lib64/ld-linux-x86-64.so.2 --help | grep x86-64-v3" --out=json 2>/dev/null) || true
while IFS= read -r node; do
[[ -z "$node" ]] && continue
result=$(jq -r --arg node "$node" '.[$node] // empty' <<< "$cpu_results" 2>/dev/null)
if [[ -z "$result" || "$result" == *"did not return"* ]]; then
offline+=("$node")
elif [[ "$result" != *"x86-64-v3 (supported"* ]]; then
# glibc appends "(supported, searched)" only when supported; the open paren keeps
# this from matching a future "(unsupported".
unsupported+=("$node")
fi
done <<< "$expected_nodes"
if [[ ${#unsupported[@]} -eq 0 && ${#offline[@]} -eq 0 ]]; then
printf "\nAll Elastic Agent nodes support x86-64-v3. We can proceed with SOUP.\n\n"
return
fi
echo ""
if [[ ${#unsupported[@]} -gt 0 ]]; then
echo "The following node(s) do NOT support the x86-64-v3 CPU instruction set:"
printf ' %s\n' "${unsupported[@]}"
echo ""
echo "Upstream Elastic now builds its binaries for x86-64-v3, so these nodes can no"
echo "longer run Elastic. Upgrading them WILL BREAK them."
echo ""
fi
if [[ ${#offline[@]} -gt 0 ]]; then
echo "The following node(s) did not respond and could not be checked:"
printf ' %s\n' "${offline[@]}"
echo ""
echo "These nodes are offline, so we cannot confirm they support x86-64-v3, which"
echo "upstream Elastic now requires."
echo ""
fi
if [[ -n $UNATTENDED ]]; then
echo "Unattended mode cannot prompt for an override. Re-run soup interactively, or pass --skip-cpu-check to bypass this check."
no_soup_for_you
fi
read -rp "Type 'override' to continue anyway, or press Enter to exit: " confirm
if [[ "${confirm,,}" == "override" ]]; then
printf "\nOverride accepted. Continuing at your own risk.\n\n"
else
no_soup_for_you
fi
}
check_fleet_server() {
echo "Checking that Elastic Fleet Server is responding."
# Modeled on the wait_for_so-elastic-fleet state check in elasticfleet/enabled.sls,
@@ -2088,6 +2168,8 @@ main() {
echo "Let's see if we need to update Security Onion."
upgrade_check
check_cpu_compatibility
upgrade_space
echo "Verifying Elasticsearch version compatibility across the grid before upgrading."
@@ -2366,6 +2448,17 @@ fi
echo "### soup has been served at $(date) ###"
}
SKIP_CPU_CHECK=false
declare -a SOUP_ARGS=()
for arg in "$@"; do
if [[ "$arg" == "--skip-cpu-check" ]]; then
SKIP_CPU_CHECK=true
else
SOUP_ARGS+=("$arg")
fi
done
set -- "${SOUP_ARGS[@]}"
while getopts ":b:f:y" opt; do
case ${opt} in
b )
@@ -2389,7 +2482,7 @@ while getopts ":b:f:y" opt; do
ISOLOC="$OPTARG"
;;
\? )
echo "Usage: soup [-b] [-y] [-f <iso location>]"
echo "Usage: soup [-b] [-y] [-f <iso location>] [--skip-cpu-check]"
exit 1
;;
: )
+2 -2
View File
@@ -54,9 +54,9 @@ so-nginx:
- /opt/so/log/nginx/:/var/log/nginx:rw
- /opt/so/tmp/nginx/:/var/lib/nginx:rw
- /opt/so/tmp/nginx/:/run:rw
- /nsm/elastic-fleet/artifacts/:/opt/socore/elastic-artifacts/artifacts:ro
{% if GLOBALS.is_manager %}
- /nsm/elastic-fleet/so_agent-installers/:/opt/socore/html/packages
- /nsm/elastic-fleet/artifacts/:/opt/socore/html/artifacts
{% if GLOBALS.is_manager %}
- /etc/pki/managerssl.crt:/etc/pki/nginx/server.crt:ro
- /etc/pki/managerssl.key:/etc/pki/nginx/server.key:ro
# ATT&CK Navigator binds
+34 -15
View File
@@ -4,8 +4,6 @@
{%- set role = grains.id.split('_') | last %}
{%- set influxpass = salt['pillar.get']('secrets:influx_pass') %}
{%- set influxauth = ('so:' + influxpass) | base64_encode %}
{%- set all_names = [GLOBALS.hostname, GLOBALS.url_base] + NGINXMERGED.ssl.alt_names %}
{%- set full_server_name = all_names | unique | join(' ') %}
worker_processes auto;
error_log /var/log/nginx/error.log;
@@ -41,31 +39,52 @@ http {
include /etc/nginx/conf.d/*.conf;
{%- if role in ['fleet'] %}
server {
listen 8443;
server_name {{ GLOBALS.hostname }};
root /opt/socore/html;
location /artifacts/ {
try_files $uri =206;
proxy_read_timeout 90;
proxy_connect_timeout 90;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Proxy "";
proxy_set_header X-Forwarded-Proto $scheme;
}
}
{%- endif %}
{%- if GLOBALS.is_manager %}
{%- set all_names = [GLOBALS.hostname, GLOBALS.url_base] + NGINXMERGED.ssl.alt_names %}
{%- set full_server_name = all_names | unique | join(' ') %}
server {
listen 80 default_server;
server_name _;
return 307 https://{{ GLOBALS.url_base }}$request_uri;
}
{%- endif %}
{%- if GLOBALS.is_manager or role == 'fleet' %}
server {
listen 8443;
{%- if GLOBALS.is_manager %}
server_name {{ full_server_name }};
{%- else %}
server_name {{ GLOBALS.hostname }};
{%- endif %}
root /opt/socore/elastic-artifacts;
root /opt/socore/html;
location /artifacts/ {
try_files $uri =404;
}
location / {
return 404;
try_files $uri =206;
proxy_read_timeout 90;
proxy_connect_timeout 90;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Proxy "";
proxy_set_header X-Forwarded-Proto $scheme;
}
}
{%- endif %}
{%- if GLOBALS.is_manager %}
server {
listen 443 ssl default_server;
http2 on;
+12 -35
View File
@@ -976,50 +976,27 @@ docker_seed_update() {
set_progress_str "$docker_seed_update_percent" "Downloading $name"
}
registry_tar_has_soversion() {
local tarfile=$1
local version=$2
[ -n "$version" ] || return 1
[ -f "$tarfile" ] || return 1
# grep -q in a pipe SIGPIPEs tar; under pipefail a match would look like failure.
grep -qF "/_manifests/tags/${version}/" < <(tar -tf "$tarfile" 2>/dev/null)
}
docker_seed_registry() {
local VERSION="$SOVERSION"
local REGISTRY_TAR=/nsm/docker-registry/docker/registry.tar
if [ -f "$REGISTRY_TAR" ] && registry_tar_has_soversion "$REGISTRY_TAR" "$VERSION"; then
info "Extracting $REGISTRY_TAR"
if ! tar xvf "$REGISTRY_TAR" -C /nsm/docker-registry/docker >> "$setup_log" 2>&1; then
error "Failed to extract $REGISTRY_TAR"
return 1
fi
logCmd "rm -f $REGISTRY_TAR"
return 0
elif [ -f "$REGISTRY_TAR" ]; then
info "Not using $REGISTRY_TAR; it has no images tagged '${VERSION}'"
if [[ "$is_airgap" == true ]]; then
error "Airgap install: $REGISTRY_TAR does not match Security Onion version '${VERSION}'. Use the ISO for this version."
return 1
fi
if [ -f /nsm/docker-registry/docker/registry.tar ]; then
logCmd "tar xvf /nsm/docker-registry/docker/registry.tar -C /nsm/docker-registry/docker"
logCmd "rm /nsm/docker-registry/docker/registry.tar"
elif [[ -d /nsm/docker-registry/docker/registry && ( -f /etc/SOCLOUD || "$is_airgap" == true ) ]]; then
echo "Using existing docker registry content"
return 0
fi
if [ "$install_type" == 'IMPORT' ]; then
container_list 'so-import'
else
container_list
fi
if [ "$install_type" == 'IMPORT' ]; then
container_list 'so-import'
else
container_list
fi
docker_seed_update_percent=25
docker_seed_update_percent=25
update_docker_containers 'netinstall' '' 'docker_seed_update' '/dev/stdout' 2>&1 | tee -a "$setup_log"
update_docker_containers 'netinstall' '' 'docker_seed_update' '/dev/stdout' 2>&1 | tee -a "$setup_log"
# Use pipe exit status of 'update_docker_containers' for return code
return ${PIPESTATUS[0]}
return ${PIPESTATUS[0]}
fi
}
elasticfleet_pillar() {
Binary file not shown.