CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2026-06-21 01:44:16 +02:00
Code Issues Packages Projects Releases Wiki Activity

Restore grouped per-integration logging and retry 409s in fleet integration loader

Browse Source 
elastic_fleet_load_integrations_dir now buffers each concurrent job's
output (header + API response) to a per-job file and prints them in
submission order after wait, restoring the readable serial-style output
while keeping concurrent writes.

Add --retry-all-errors to the integration create/update curl calls so
transient 409 conflicts from concurrent writes to the same agent policy
are retried (curl --retry alone does not retry 409).
This commit is contained in:
Josh Patterson
2026-06-18 11:19:36 -04:00
parent 62c01a9756
commit d0bea2ebcb
1 changed files with 22 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
salt/elasticfleet/tools/sbin/so-elastic-fleet-common
+22 -6
View File
@@ -53,9 +53,13 @@ elastic_fleet_load_integrations_dir() {
local DIR=$2
local LABEL=$3
local SKIP_CREATE_NAME=$4
local POLICY_JSON FAIL_FILE INTEGRATION NAME ID
local POLICY_JSON FAIL_FILE OUT_DIR INTEGRATION NAME ID i
FAIL_FILE=$(mktemp)
# Each job buffers its full output (header + API response) into its own file so the
# parent can print them grouped and in submission order after concurrent writes finish.
OUT_DIR=$(mktemp -d)
i=0
# Fetch the agent policy a single time; we look up integration ids locally below.
POLICY_JSON=$(fleet_api "agent_policies/$AGENT_POLICY")
@@ -67,23 +71,31 @@ elastic_fleet_load_integrations_dir() {
elastic_fleet_throttle
{
local RESP
if [ -n "$ID" ]; then
printf "\n\n%s - Updating integration %s\n" "$LABEL" "$NAME"
if ! elastic_fleet_integration_update "$ID" "@$INTEGRATION"; then
if ! RESP=$(elastic_fleet_integration_update "$ID" "@$INTEGRATION"); then
flock 9; echo "update ${INTEGRATION##*/}" >&9
fi
printf '%s\n' "$RESP"
elif [ -n "$SKIP_CREATE_NAME" ] && [ "$NAME" == "$SKIP_CREATE_NAME" ]; then
printf "\n\n%s - Skipping creation of %s\n" "$LABEL" "$NAME"
else
printf "\n\n%s - Creating integration %s\n" "$LABEL" "$NAME"
if ! elastic_fleet_integration_create "@$INTEGRATION"; then
if ! RESP=$(elastic_fleet_integration_create "@$INTEGRATION"); then
flock 9; echo "create ${INTEGRATION##*/}" >&9
fi
printf '%s\n' "$RESP"
fi
} 9>>"$FAIL_FILE" &
} >"$OUT_DIR/$(printf '%03d' "$i")" 9>>"$FAIL_FILE" &
i=$((i+1))
done
wait
# Emit per-integration output grouped and in submission order (glob sorts numerically).
cat "$OUT_DIR"/* 2>/dev/null
rm -rf "$OUT_DIR"
local rc=0
if [ -s "$FAIL_FILE" ]; then
printf "\n%s: failed integrations:\n" "$LABEL"
@@ -110,7 +122,9 @@ elastic_fleet_integration_create() {
JSON_STRING=$1
if ! fleet_api "package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPOST -d "$JSON_STRING"; then
# --retry-all-errors so transient 409 conflicts (concurrent writes to the same agent
# policy) are retried; curl --retry alone does not retry 409.
if ! fleet_api "package_policies" --retry-all-errors -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPOST -d "$JSON_STRING"; then
return 1
fi
}
@@ -141,7 +155,9 @@ elastic_fleet_integration_update() {
JSON_STRING=$2
if ! fleet_api "package_policies/$UPDATE_ID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPUT -d "$JSON_STRING"; then
# --retry-all-errors so transient 409 conflicts (concurrent writes to the same agent
# policy) are retried; curl --retry alone does not retry 409.
if ! fleet_api "package_policies/$UPDATE_ID" --retry-all-errors -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPUT -d "$JSON_STRING"; then
return 1
fi
}