mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2026-06-12 13:19:22 +02:00
Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b1273573ed | |||
| 6c42c419e2 | |||
| f23652397c | |||
| 07d3b148b5 | |||
| 780d9faf0d | |||
| d2fe51d5fe | |||
| 83aaa76f98 |
@@ -101,17 +101,6 @@ so-elastic-fleet:
|
|||||||
- file: trusttheca
|
- file: trusttheca
|
||||||
- x509: etc_elasticfleet_key
|
- x509: etc_elasticfleet_key
|
||||||
- x509: etc_elasticfleet_crt
|
- x509: etc_elasticfleet_crt
|
||||||
|
|
||||||
wait_for_so-elastic-fleet:
|
|
||||||
http.wait_for_successful_query:
|
|
||||||
- name: "https://localhost:8220/api/status"
|
|
||||||
- ssl: True
|
|
||||||
- verify_ssl: False
|
|
||||||
- status: 200
|
|
||||||
- wait_for: 300
|
|
||||||
- request_interval: 15
|
|
||||||
- require:
|
|
||||||
- docker_container: so-elastic-fleet
|
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
delete_so-elastic-fleet_so-status.disabled:
|
delete_so-elastic-fleet_so-status.disabled:
|
||||||
|
|||||||
@@ -9,7 +9,6 @@
|
|||||||
|
|
||||||
include:
|
include:
|
||||||
- elasticfleet.config
|
- elasticfleet.config
|
||||||
- kibana.enabled
|
|
||||||
|
|
||||||
# If enabled, automatically update Fleet Logstash Outputs
|
# If enabled, automatically update Fleet Logstash Outputs
|
||||||
{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration %}
|
{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration %}
|
||||||
@@ -20,8 +19,6 @@ so-elastic-fleet-auto-configure-logstash-outputs:
|
|||||||
- retry:
|
- retry:
|
||||||
attempts: 4
|
attempts: 4
|
||||||
interval: 30
|
interval: 30
|
||||||
- require:
|
|
||||||
- http: wait_for_so-kibana
|
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
# If enabled, automatically update Fleet Server URLs & ES Connection
|
# If enabled, automatically update Fleet Server URLs & ES Connection
|
||||||
@@ -31,8 +28,6 @@ so-elastic-fleet-auto-configure-server-urls:
|
|||||||
- retry:
|
- retry:
|
||||||
attempts: 4
|
attempts: 4
|
||||||
interval: 30
|
interval: 30
|
||||||
- require:
|
|
||||||
- http: wait_for_so-kibana
|
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
# Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs
|
# Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs
|
||||||
@@ -42,8 +37,6 @@ so-elastic-fleet-auto-configure-elasticsearch-urls:
|
|||||||
- retry:
|
- retry:
|
||||||
attempts: 4
|
attempts: 4
|
||||||
interval: 30
|
interval: 30
|
||||||
- require:
|
|
||||||
- http: wait_for_so-kibana
|
|
||||||
|
|
||||||
so-elastic-fleet-auto-configure-artifact-urls:
|
so-elastic-fleet-auto-configure-artifact-urls:
|
||||||
cmd.run:
|
cmd.run:
|
||||||
@@ -51,8 +44,6 @@ so-elastic-fleet-auto-configure-artifact-urls:
|
|||||||
- retry:
|
- retry:
|
||||||
attempts: 4
|
attempts: 4
|
||||||
interval: 30
|
interval: 30
|
||||||
- require:
|
|
||||||
- http: wait_for_so-kibana
|
|
||||||
|
|
||||||
so-elastic-fleet-package-statefile:
|
so-elastic-fleet-package-statefile:
|
||||||
file.managed:
|
file.managed:
|
||||||
@@ -64,9 +55,7 @@ so-elastic-fleet-package-upgrade:
|
|||||||
- name: /usr/sbin/so-elastic-fleet-package-upgrade
|
- name: /usr/sbin/so-elastic-fleet-package-upgrade
|
||||||
- retry:
|
- retry:
|
||||||
attempts: 3
|
attempts: 3
|
||||||
interval: 30
|
interval: 10
|
||||||
- require:
|
|
||||||
- http: wait_for_so-kibana
|
|
||||||
- onchanges:
|
- onchanges:
|
||||||
- file: /opt/so/state/elastic_fleet_packages.txt
|
- file: /opt/so/state/elastic_fleet_packages.txt
|
||||||
|
|
||||||
@@ -76,8 +65,6 @@ so-elastic-fleet-integrations:
|
|||||||
- retry:
|
- retry:
|
||||||
attempts: 3
|
attempts: 3
|
||||||
interval: 10
|
interval: 10
|
||||||
- require:
|
|
||||||
- http: wait_for_so-kibana
|
|
||||||
|
|
||||||
so-elastic-agent-grid-upgrade:
|
so-elastic-agent-grid-upgrade:
|
||||||
cmd.run:
|
cmd.run:
|
||||||
@@ -85,8 +72,6 @@ so-elastic-agent-grid-upgrade:
|
|||||||
- retry:
|
- retry:
|
||||||
attempts: 12
|
attempts: 12
|
||||||
interval: 5
|
interval: 5
|
||||||
- require:
|
|
||||||
- http: wait_for_so-kibana
|
|
||||||
|
|
||||||
so-elastic-fleet-integration-upgrade:
|
so-elastic-fleet-integration-upgrade:
|
||||||
cmd.run:
|
cmd.run:
|
||||||
@@ -94,22 +79,16 @@ so-elastic-fleet-integration-upgrade:
|
|||||||
- retry:
|
- retry:
|
||||||
attempts: 3
|
attempts: 3
|
||||||
interval: 10
|
interval: 10
|
||||||
- require:
|
|
||||||
- http: wait_for_so-kibana
|
|
||||||
|
|
||||||
{# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #}
|
{# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #}
|
||||||
so-elastic-fleet-addon-integrations:
|
so-elastic-fleet-addon-integrations:
|
||||||
cmd.run:
|
cmd.run:
|
||||||
- name: /usr/sbin/so-elastic-fleet-optional-integrations-load
|
- name: /usr/sbin/so-elastic-fleet-optional-integrations-load
|
||||||
- require:
|
|
||||||
- http: wait_for_so-kibana
|
|
||||||
|
|
||||||
{% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %}
|
{% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %}
|
||||||
so-elastic-defend-manage-filters-file-watch:
|
so-elastic-defend-manage-filters-file-watch:
|
||||||
cmd.run:
|
cmd.run:
|
||||||
- name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log
|
- name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log
|
||||||
- require:
|
|
||||||
- http: wait_for_so-kibana
|
|
||||||
- onchanges:
|
- onchanges:
|
||||||
- file: elasticdefendcustom
|
- file: elasticdefendcustom
|
||||||
- file: elasticdefenddisabled
|
- file: elasticdefenddisabled
|
||||||
|
|||||||
@@ -108,12 +108,9 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
|
|||||||
done
|
done
|
||||||
|
|
||||||
# Only create the state file if all policies were created/updated successfully
|
# Only create the state file if all policies were created/updated successfully
|
||||||
if [[ $RETURN_CODE -eq 0 ]]; then
|
if [[ "$RETURN_CODE" != "1" ]]; then
|
||||||
touch /opt/so/state/eaintegrations.txt
|
touch /opt/so/state/eaintegrations.txt
|
||||||
else
|
|
||||||
exit 1
|
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
echo "Fleet integration policies already loaded."
|
exit $RETURN_CODE
|
||||||
exit 0
|
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -16,7 +16,6 @@
|
|||||||
STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt
|
STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt
|
||||||
INSTALLED_PACKAGE_LIST=/tmp/esfleet_installed_packages.json
|
INSTALLED_PACKAGE_LIST=/tmp/esfleet_installed_packages.json
|
||||||
BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json
|
BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json
|
||||||
BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json
|
|
||||||
BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json
|
BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json
|
||||||
INTEGRATION_PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
|
INTEGRATION_PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
|
||||||
INPUT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_input_package_components.json
|
INPUT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_input_package_components.json
|
||||||
@@ -29,29 +28,6 @@ PENDING_UPDATE=false
|
|||||||
# Requiring some level of manual Elastic Stack configuration before installation
|
# Requiring some level of manual Elastic Stack configuration before installation
|
||||||
EXCLUDED_INTEGRATIONS=('apm')
|
EXCLUDED_INTEGRATIONS=('apm')
|
||||||
|
|
||||||
version_conversion(){
|
|
||||||
version=$1
|
|
||||||
echo "$version" | awk -F '.' '{ printf("%d%03d%03d\n", $1, $2, $3); }'
|
|
||||||
}
|
|
||||||
|
|
||||||
compare_versions() {
|
|
||||||
version1=$1
|
|
||||||
version2=$2
|
|
||||||
|
|
||||||
# Convert versions to numbers
|
|
||||||
num1=$(version_conversion "$version1")
|
|
||||||
num2=$(version_conversion "$version2")
|
|
||||||
|
|
||||||
# Compare using bc
|
|
||||||
if (( $(echo "$num1 < $num2" | bc -l) )); then
|
|
||||||
echo "less"
|
|
||||||
elif (( $(echo "$num1 > $num2" | bc -l) )); then
|
|
||||||
echo "greater"
|
|
||||||
else
|
|
||||||
echo "equal"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
IFS=$'\n'
|
IFS=$'\n'
|
||||||
agent_policies=$(elastic_fleet_agent_policy_ids)
|
agent_policies=$(elastic_fleet_agent_policy_ids)
|
||||||
if [ $? -ne 0 ]; then
|
if [ $? -ne 0 ]; then
|
||||||
@@ -63,23 +39,23 @@ default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.l
|
|||||||
|
|
||||||
in_use_integrations=()
|
in_use_integrations=()
|
||||||
|
|
||||||
|
# Fetch each agent policy once; its package_policies[] already contain both the integration name
|
||||||
|
# and the .package.name, so extract all non-default package names locally in a single jq instead
|
||||||
|
# of re-fetching the same policy per integration.
|
||||||
|
default_packages_json=$(printf '%s\n' "${default_packages[@]}" | jq -R . | jq -s '.')
|
||||||
for AGENT_POLICY in $agent_policies; do
|
for AGENT_POLICY in $agent_policies; do
|
||||||
|
|
||||||
if ! integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY"); then
|
if ! policy_json=$(fleet_api "agent_policies/$AGENT_POLICY"); then
|
||||||
# skip the agent policy if we can't get required info, let salt retry. Integrations loaded by this script are non-default integrations.
|
# skip the agent policy if we can't get required info, let salt retry. Integrations loaded by this script are non-default integrations.
|
||||||
echo "Skipping $AGENT_POLICY.. "
|
echo "Skipping $AGENT_POLICY.. "
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
for INTEGRATION in $integrations; do
|
# non-default integrations that are in-use in any policy
|
||||||
if ! PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION"); then
|
while IFS= read -r PACKAGE_NAME; do
|
||||||
echo "Not adding $INTEGRATION, couldn't get package name"
|
[ -n "$PACKAGE_NAME" ] && in_use_integrations+=("$PACKAGE_NAME")
|
||||||
continue
|
done < <(jq -r --argjson defaults "$default_packages_json" \
|
||||||
fi
|
'.item.package_policies[].package.name | select(. as $n | ($defaults | index($n)) | not)' \
|
||||||
# non-default integrations that are in-use in any policy
|
<<<"$policy_json")
|
||||||
if ! [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then
|
|
||||||
in_use_integrations+=("$PACKAGE_NAME")
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
done
|
done
|
||||||
|
|
||||||
if [[ -f $STATE_FILE_SUCCESS ]]; then
|
if [[ -f $STATE_FILE_SUCCESS ]]; then
|
||||||
@@ -90,72 +66,55 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then
|
|||||||
rm -f $INSTALLED_PACKAGE_LIST
|
rm -f $INSTALLED_PACKAGE_LIST
|
||||||
echo $latest_package_list | jq '{packages: [.items[] | {name: .name, latest_version: .version, installed_version: .installationInfo.version, subscription: .conditions.elastic.subscription }]}' >> $INSTALLED_PACKAGE_LIST
|
echo $latest_package_list | jq '{packages: [.items[] | {name: .name, latest_version: .version, installed_version: .installationInfo.version, subscription: .conditions.elastic.subscription }]}' >> $INSTALLED_PACKAGE_LIST
|
||||||
|
|
||||||
while read -r package; do
|
# Build the bulk install list and the per-package status messages with two jq passes
|
||||||
# get package details
|
# instead of a per-package bash loop. The old loop forked ~10 processes per package
|
||||||
package_name=$(echo "$package" | jq -r '.name')
|
# (5 jq + awk/bc for the version compare) and re-parsed/rewrote a growing JSON file on
|
||||||
latest_version=$(echo "$package" | jq -r '.latest_version')
|
# every add (O(n^2)). Selection and messages below are identical to that logic.
|
||||||
installed_version=$(echo "$package" | jq -r '.installed_version')
|
SUB={% if SUB %}true{% else %}false{% endif %}
|
||||||
subscription=$(echo "$package" | jq -r '.subscription')
|
AUTOUP={% if AUTO_UPGRADE_INTEGRATIONS %}true{% else %}false{% endif %}
|
||||||
bulk_package=$(echo "$package" | jq '{name: .name, version: .latest_version}' )
|
EXCLUDED_JSON=$(printf '%s\n' "${EXCLUDED_INTEGRATIONS[@]}" | jq -R 'select(length>0)' | jq -s '.')
|
||||||
|
INUSE_JSON=$(printf '%s\n' "${in_use_integrations[@]}" | jq -R 'select(length>0)' | jq -s 'unique')
|
||||||
|
|
||||||
if [[ ! "${EXCLUDED_INTEGRATIONS[@]}" =~ "$package_name" ]]; then
|
# vnum replicates the previous version_conversion (%d%03d%03d of the first three dotted
|
||||||
{% if not SUB %}
|
# fields); needs() replicates the excluded/subscription/installed/upgrade/in-use logic.
|
||||||
if [[ "$subscription" != "basic" && "$subscription" != "null" && -n "$subscription" ]]; then
|
JQ_DECISION='
|
||||||
# pass over integrations that require non-basic elastic license
|
def vnum:
|
||||||
echo "$package_name integration requires an Elastic license of $subscription or greater... skipping"
|
[ (split(".")|.[0:3][] | gsub("[^0-9].*";"") | (if .=="" then "0" else . end) | tonumber) ]
|
||||||
continue
|
| (.[0]//0)*1000000 + (.[1]//0)*1000 + (.[2]//0);
|
||||||
else
|
def needs($sub;$autoup;$excluded;$inuse):
|
||||||
if [[ "$installed_version" == "null" || -z "$installed_version" ]]; then
|
.name as $n
|
||||||
echo "$package_name is not installed... Adding to next update."
|
| ($n | IN($excluded[]) | not)
|
||||||
jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
|
and ( $sub or (.subscription==null or .subscription=="basic" or .subscription=="") )
|
||||||
|
and ( (.installed_version==null or .installed_version=="")
|
||||||
|
or ( ((.latest_version|vnum) > (.installed_version|vnum))
|
||||||
|
and ( $autoup or ($n | IN($inuse[]) | not) ) ) );'
|
||||||
|
|
||||||
PENDING_UPDATE=true
|
JQ_ARGS=(--argjson sub "$SUB" --argjson autoup "$AUTOUP" --argjson excluded "$EXCLUDED_JSON" --argjson inuse "$INUSE_JSON")
|
||||||
else
|
|
||||||
results=$(compare_versions "$latest_version" "$installed_version")
|
|
||||||
if [ $results == "greater" ]; then
|
|
||||||
{#- When auto_upgrade_integrations is false, skip upgrading in_use_integrations #}
|
|
||||||
{%- if not AUTO_UPGRADE_INTEGRATIONS %}
|
|
||||||
if ! [[ " ${in_use_integrations[@]} " =~ " $package_name " ]]; then
|
|
||||||
{%- endif %}
|
|
||||||
echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update."
|
|
||||||
jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
|
|
||||||
|
|
||||||
PENDING_UPDATE=true
|
# (a) Per-package status messages (parity with the previous echo output).
|
||||||
{%- if not AUTO_UPGRADE_INTEGRATIONS %}
|
jq -r "${JQ_ARGS[@]}" "$JQ_DECISION"'
|
||||||
else
|
.packages[]
|
||||||
echo "skipping available upgrade for in use integration - $package_name."
|
| .name as $n
|
||||||
fi
|
| if ($n|IN($excluded[])) then "Skipping \($n)..."
|
||||||
{%- endif %}
|
elif (($sub|not) and (.subscription!=null and .subscription!="basic" and .subscription!="")) then
|
||||||
fi
|
"\($n) integration requires an Elastic license of \(.subscription) or greater... skipping"
|
||||||
fi
|
elif (.installed_version==null or .installed_version=="") then
|
||||||
fi
|
"\($n) is not installed... Adding to next update."
|
||||||
{% else %}
|
elif ((.latest_version|vnum) > (.installed_version|vnum)) then
|
||||||
if [[ "$installed_version" == "null" || -z "$installed_version" ]]; then
|
(if ($autoup or ($n|IN($inuse[])|not))
|
||||||
echo "$package_name is not installed... Adding to next update."
|
then "\($n) is at version \(.installed_version) latest version is \(.latest_version)... Adding to next update."
|
||||||
jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
|
else "skipping available upgrade for in use integration - \($n)." end)
|
||||||
PENDING_UPDATE=true
|
else empty end
|
||||||
else
|
' "$INSTALLED_PACKAGE_LIST"
|
||||||
results=$(compare_versions "$latest_version" "$installed_version")
|
|
||||||
if [ $results == "greater" ]; then
|
# (b) The bulk install list, built in a single pass.
|
||||||
{#- When auto_upgrade_integrations is false, skip upgrading in_use_integrations #}
|
jq "${JQ_ARGS[@]}" "$JQ_DECISION"'
|
||||||
{%- if not AUTO_UPGRADE_INTEGRATIONS %}
|
{packages: [ .packages[] | select(needs($sub;$autoup;$excluded;$inuse)) | {name, version: .latest_version} ]}
|
||||||
if ! [[ " ${in_use_integrations[@]} " =~ " $package_name " ]]; then
|
' "$INSTALLED_PACKAGE_LIST" > "$BULK_INSTALL_PACKAGE_LIST"
|
||||||
{%- endif %}
|
|
||||||
echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update."
|
if jq -e '.packages | length > 0' "$BULK_INSTALL_PACKAGE_LIST" >/dev/null; then
|
||||||
jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
|
PENDING_UPDATE=true
|
||||||
PENDING_UPDATE=true
|
fi
|
||||||
{%- if not AUTO_UPGRADE_INTEGRATIONS %}
|
|
||||||
else
|
|
||||||
echo "skipping available upgrade for in use integration - $package_name."
|
|
||||||
fi
|
|
||||||
{%- endif %}
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
{% endif %}
|
|
||||||
else
|
|
||||||
echo "Skipping $package_name..."
|
|
||||||
fi
|
|
||||||
done <<< "$(jq -c '.packages[]' "$INSTALLED_PACKAGE_LIST")"
|
|
||||||
|
|
||||||
if [ "$PENDING_UPDATE" = true ]; then
|
if [ "$PENDING_UPDATE" = true ]; then
|
||||||
# Run chunked install of packages
|
# Run chunked install of packages
|
||||||
|
|||||||
@@ -8,33 +8,18 @@
|
|||||||
|
|
||||||
. /usr/sbin/so-elastic-fleet-common
|
. /usr/sbin/so-elastic-fleet-common
|
||||||
|
|
||||||
PKG_LOAD_FAILURES=0
|
|
||||||
PKG_LOAD_FAILURES_NAMES=()
|
|
||||||
|
|
||||||
{%- for PACKAGE in SUPPORTED_PACKAGES %}
|
{%- for PACKAGE in SUPPORTED_PACKAGES %}
|
||||||
echo "Upgrading {{ PACKAGE }} package..."
|
echo "Upgrading {{ PACKAGE }} package..."
|
||||||
if VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
|
if VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
|
||||||
if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then
|
if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then
|
||||||
PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
|
# exit 1 on failure to upgrade a default package, allow salt to handle retries
|
||||||
PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
|
echo -e "\nERROR: Failed to upgrade $PACKAGE to version: $VERSION"
|
||||||
|
exit 1
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
|
echo -e "\nERROR: Failed to get version information for integration $PACKAGE"
|
||||||
PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
|
|
||||||
fi
|
fi
|
||||||
echo
|
echo
|
||||||
{%- endfor %}
|
{%- endfor %}
|
||||||
|
|
||||||
if [ $PKG_LOAD_FAILURES -gt 0 ]; then
|
|
||||||
echo "ERROR: Failed to upgrade $PKG_LOAD_FAILURES package(s):"
|
|
||||||
for PKG in "${PKG_LOAD_FAILURES_NAMES[@]}"; do
|
|
||||||
echo " - $PKG"
|
|
||||||
done
|
|
||||||
# exit 1 on failure to upgrade a default package, allow salt to handle retries
|
|
||||||
exit 1
|
|
||||||
else
|
|
||||||
echo "Successfully upgraded all packages."
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo
|
echo
|
||||||
/usr/sbin/so-elasticsearch-templates-load
|
/usr/sbin/so-elasticsearch-templates-load
|
||||||
|
|||||||
@@ -6,6 +6,37 @@
|
|||||||
|
|
||||||
. /usr/sbin/so-common
|
. /usr/sbin/so-common
|
||||||
|
|
||||||
|
MAX_JOBS=10
|
||||||
|
|
||||||
|
# Lock used to serialize block writes so concurrent jobs never interleave their output.
|
||||||
|
ILM_OUTPUT_LOCK=$(mktemp)
|
||||||
|
trap 'rm -f "$ILM_OUTPUT_LOCK"' EXIT
|
||||||
|
|
||||||
|
# Policies are loaded concurrently (up to MAX_JOBS at a time) for speed. Each policy's block is
|
||||||
|
# printed the moment its curl returns, so output appears in COMPLETION ORDER, not the order
|
||||||
|
# policies are defined in configuration.
|
||||||
|
echo "Loading ILM policies concurrently; output below appears in completion order, not configuration order."
|
||||||
|
echo
|
||||||
|
|
||||||
|
put_policy() {
|
||||||
|
local desc="$1" policyname="$2" data="$3" result
|
||||||
|
result=$(curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L \
|
||||||
|
-X PUT "https://localhost:9200/_ilm/policy/${policyname}" \
|
||||||
|
-H 'Content-Type: application/json' -d"${data}")
|
||||||
|
# curl above ran in parallel; serialize just this block write so concurrent jobs never interleave.
|
||||||
|
{
|
||||||
|
flock 200
|
||||||
|
printf 'Setting up %s policy...\n%s\n\n' "${desc}" "${result}"
|
||||||
|
} 200>>"${ILM_OUTPUT_LOCK}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Block until fewer than MAX_JOBS background curls are running.
|
||||||
|
throttle() {
|
||||||
|
while (( $(jobs -rp | wc -l) >= MAX_JOBS )); do
|
||||||
|
wait -n
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
{%- from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
|
{%- from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
|
||||||
{%- if GLOBALS.role != "so-heavynode" %}
|
{%- if GLOBALS.role != "so-heavynode" %}
|
||||||
{%- from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %}
|
{%- from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %}
|
||||||
@@ -14,35 +45,26 @@
|
|||||||
{%- for index, settings in ES_INDEX_SETTINGS.items() %}
|
{%- for index, settings in ES_INDEX_SETTINGS.items() %}
|
||||||
{%- if settings.policy is defined %}
|
{%- if settings.policy is defined %}
|
||||||
{%- if index == 'so-logs-detections.alerts' %}
|
{%- if index == 'so-logs-detections.alerts' %}
|
||||||
echo
|
throttle
|
||||||
echo "Setting up so-logs-detections.alerts-so policy..."
|
put_policy "so-logs-detections.alerts-so" "{{ index }}-so" '{ "policy": {{ settings.policy | tojson(true) }} }' &
|
||||||
curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-so" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
|
|
||||||
echo
|
|
||||||
{%- elif index == 'so-logs-soc' %}
|
{%- elif index == 'so-logs-soc' %}
|
||||||
echo
|
throttle
|
||||||
echo "Setting up so-soc-logs policy..."
|
put_policy "so-soc-logs" "so-soc-logs" '{ "policy": {{ settings.policy | tojson(true) }} }' &
|
||||||
curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/so-soc-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
|
throttle
|
||||||
echo
|
put_policy "{{ index }}-logs" "{{ index }}-logs" '{ "policy": {{ settings.policy | tojson(true) }} }' &
|
||||||
echo
|
|
||||||
echo "Setting up {{ index }}-logs policy..."
|
|
||||||
curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
|
|
||||||
echo
|
|
||||||
{%- else %}
|
{%- else %}
|
||||||
echo
|
throttle
|
||||||
echo "Setting up {{ index }}-logs policy..."
|
put_policy "{{ index }}-logs" "{{ index }}-logs" '{ "policy": {{ settings.policy | tojson(true) }} }' &
|
||||||
curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
|
|
||||||
echo
|
|
||||||
{%- endif %}
|
{%- endif %}
|
||||||
{%- endif %}
|
{%- endif %}
|
||||||
{%- endfor %}
|
{%- endfor %}
|
||||||
echo
|
|
||||||
{%- if GLOBALS.role != "so-heavynode" %}
|
{%- if GLOBALS.role != "so-heavynode" %}
|
||||||
{%- for index, settings in ALL_ADDON_SETTINGS.items() %}
|
{%- for index, settings in ALL_ADDON_SETTINGS.items() %}
|
||||||
{%- if settings.policy is defined %}
|
{%- if settings.policy is defined %}
|
||||||
echo
|
throttle
|
||||||
echo "Setting up {{ index }}-logs policy..."
|
put_policy "{{ index }}-logs" "{{ index }}-logs" '{ "policy": {{ settings.policy | tojson(true) }} }' &
|
||||||
curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
|
|
||||||
echo
|
|
||||||
{%- endif %}
|
{%- endif %}
|
||||||
{%- endfor %}
|
{%- endfor %}
|
||||||
{%- endif %}
|
{%- endif %}
|
||||||
|
|
||||||
|
wait
|
||||||
|
|||||||
@@ -6,7 +6,6 @@
|
|||||||
{% from 'allowed_states.map.jinja' import allowed_states %}
|
{% from 'allowed_states.map.jinja' import allowed_states %}
|
||||||
{% if sls.split('.')[0] in allowed_states %}
|
{% if sls.split('.')[0] in allowed_states %}
|
||||||
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
|
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
|
||||||
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
|
|
||||||
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
||||||
|
|
||||||
include:
|
include:
|
||||||
@@ -61,19 +60,6 @@ so-kibana:
|
|||||||
- watch:
|
- watch:
|
||||||
- file: kibanaconfig
|
- file: kibanaconfig
|
||||||
|
|
||||||
wait_for_so-kibana:
|
|
||||||
http.wait_for_successful_query:
|
|
||||||
- name: "http://localhost:5601/api/status"
|
|
||||||
- username: 'so_elastic'
|
|
||||||
- password: '{{ ELASTICSEARCHMERGED.auth.users.so_elastic_user.pass }}'
|
|
||||||
- ssl: True
|
|
||||||
- verify_ssl: False
|
|
||||||
- status: 200
|
|
||||||
- wait_for: 300
|
|
||||||
- request_interval: 15
|
|
||||||
- require:
|
|
||||||
- docker_container: so-kibana
|
|
||||||
|
|
||||||
delete_so-kibana_so-status.disabled:
|
delete_so-kibana_so-status.disabled:
|
||||||
file.uncomment:
|
file.uncomment:
|
||||||
- name: /opt/so/conf/so-status/so-status.conf
|
- name: /opt/so/conf/so-status/so-status.conf
|
||||||
|
|||||||
@@ -343,10 +343,11 @@ highstate() {
|
|||||||
masterlock() {
|
masterlock() {
|
||||||
echo "Locking Salt Master"
|
echo "Locking Salt Master"
|
||||||
mv -v $TOPFILE $BACKUPTOPFILE
|
mv -v $TOPFILE $BACKUPTOPFILE
|
||||||
echo "base:" > $TOPFILE
|
# Render the real top file only for the host running soup; every other
|
||||||
echo " $MINIONID:" >> $TOPFILE
|
# minion gets an empty top (no states) while the master is upgrading.
|
||||||
echo " - ca" >> $TOPFILE
|
echo "{% if grains['id'] == '$MINIONID' %}" > $TOPFILE
|
||||||
echo " - elasticsearch" >> $TOPFILE
|
cat $BACKUPTOPFILE >> $TOPFILE
|
||||||
|
echo "{% endif %}" >> $TOPFILE
|
||||||
}
|
}
|
||||||
|
|
||||||
masterunlock() {
|
masterunlock() {
|
||||||
|
|||||||
Reference in New Issue
Block a user