mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2026-06-06 18:35:26 +02:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 reyesj2
|
c98193879d |
+11
-11
@@ -1,17 +1,17 @@
|
||||
### 3.1.0-20260528 ISO image released on 2026/05/28
|
||||
### 3.0.0-20260331 ISO image released on 2026/03/31
|
||||
|
||||
|
||||
### Download and Verify
|
||||
|
||||
3.1.0-20260528 ISO image:
|
||||
https://download.securityonion.net/file/securityonion/securityonion-3.1.0-20260528.iso
|
||||
3.0.0-20260331 ISO image:
|
||||
https://download.securityonion.net/file/securityonion/securityonion-3.0.0-20260331.iso
|
||||
|
||||
MD5: 9D6FF58DEEE24089D722C73169765B3E
|
||||
SHA1: 2B8B816B6CEC3B7F96B3C5E040EBF502DD2C412F
|
||||
SHA256: 62FAB57E247C843D6A04F0796D8162C732B65D82FC3E4A59D087135B9FD32912
|
||||
MD5: ECD318A1662A6FDE0EF213F5A9BD4B07
|
||||
SHA1: E55BE314440CCF3392DC0B06BC5E270B43176D9C
|
||||
SHA256: 7FC47405E335CBE5C2B6C51FE7AC60248F35CBE504907B8B5A33822B23F8F4D5
|
||||
|
||||
Signature for ISO image:
|
||||
https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.1.0-20260528.iso.sig
|
||||
https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.0.0-20260331.iso.sig
|
||||
|
||||
Signing key:
|
||||
https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/main/KEYS
|
||||
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/
|
||||
|
||||
Download the signature file for the ISO:
|
||||
```
|
||||
wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.1.0-20260528.iso.sig
|
||||
wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.0.0-20260331.iso.sig
|
||||
```
|
||||
|
||||
Download the ISO image:
|
||||
```
|
||||
wget https://download.securityonion.net/file/securityonion/securityonion-3.1.0-20260528.iso
|
||||
wget https://download.securityonion.net/file/securityonion/securityonion-3.0.0-20260331.iso
|
||||
```
|
||||
|
||||
Verify the downloaded ISO image using the signature file:
|
||||
```
|
||||
gpg --verify securityonion-3.1.0-20260528.iso.sig securityonion-3.1.0-20260528.iso
|
||||
gpg --verify securityonion-3.0.0-20260331.iso.sig securityonion-3.0.0-20260331.iso
|
||||
```
|
||||
|
||||
The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
|
||||
```
|
||||
gpg: Signature made Wed 27 May 2026 03:03:59 PM EDT using RSA key ID FE507013
|
||||
gpg: Signature made Mon 30 Mar 2026 06:22:14 PM EDT using RSA key ID FE507013
|
||||
gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
|
||||
gpg: WARNING: This key is not certified with a trusted signature!
|
||||
gpg: There is no indication that the signature belongs to the owner.
|
||||
|
||||
@@ -26,33 +26,14 @@ commonpkgs:
|
||||
- net-tools
|
||||
- nmap-ncat
|
||||
- procps-ng
|
||||
{# OL10 test path: python3-docker / python3-m2crypto are not packaged in EPEL 10 and are not
|
||||
referenced by SO code (salt uses its bundled docker module from salt/python_modules.sls).
|
||||
python3-rich is also unavailable on EL10 (its pygments dep is not packaged), so it is
|
||||
installed via pip below. Gate on the grain because GLOBALS/pillars are not available this
|
||||
early (see header note). #}
|
||||
{% if grains['osmajorrelease']|int < 10 %}
|
||||
- python3-docker
|
||||
- python3-m2crypto
|
||||
- python3-rich
|
||||
{% else %}
|
||||
- python3-pip
|
||||
{% endif %}
|
||||
- python3-packaging
|
||||
- python3-pyyaml
|
||||
- python3-rich
|
||||
- rsync
|
||||
- sqlite
|
||||
- tcpdump
|
||||
- unzip
|
||||
- wget
|
||||
- yum-utils
|
||||
|
||||
{% if grains['osmajorrelease']|int >= 10 %}
|
||||
# OL10 test path: rich is not packaged for EL10; install it into the system python3 for so-status.
|
||||
commonpkgs_pip_rich:
|
||||
cmd.run:
|
||||
- name: python3 -m pip install rich
|
||||
- unless: python3 -c "import rich"
|
||||
- require:
|
||||
- pkg: commonpkgs
|
||||
{% endif %}
|
||||
|
||||
@@ -354,12 +354,7 @@ gpg_rpm_import() {
|
||||
else
|
||||
local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/$OS/keys"
|
||||
fi
|
||||
if [[ "$OSVER" == "10" ]]; then
|
||||
# OL10 test path uses public repos; the public oracle-epel-release and docker repos provide their own keys
|
||||
RPMKEYS=('RPM-GPG-KEY-oracle' 'SALT-PROJECT-GPG-PUBKEY-2023.pub')
|
||||
else
|
||||
RPMKEYS=('RPM-GPG-KEY-oracle' 'RPM-GPG-KEY-EPEL-9' 'SALT-PROJECT-GPG-PUBKEY-2023.pub' 'docker.pub' 'securityonion.pub')
|
||||
fi
|
||||
RPMKEYS=('RPM-GPG-KEY-oracle' 'RPM-GPG-KEY-EPEL-9' 'SALT-PROJECT-GPG-PUBKEY-2023.pub' 'docker.pub' 'securityonion.pub')
|
||||
for RPMKEY in "${RPMKEYS[@]}"; do
|
||||
rpm --import $RPMKEYSLOC/$RPMKEY
|
||||
echo "Imported $RPMKEY"
|
||||
@@ -631,9 +626,9 @@ salt_minion_count() {
|
||||
}
|
||||
|
||||
set_os() {
|
||||
if [ -f /etc/oracle-release ] && grep -qE "release (9|10)\b" /etc/oracle-release; then
|
||||
if [ -f /etc/redhat-release ] && grep -q "Red Hat Enterprise Linux release 9" /etc/redhat-release && [ -f /etc/oracle-release ]; then
|
||||
OS=oracle
|
||||
OSVER=$(grep -oE "release [0-9]+" /etc/oracle-release | grep -oE "[0-9]+")
|
||||
OSVER=9
|
||||
is_oracle=true
|
||||
is_rpm=true
|
||||
fi
|
||||
|
||||
@@ -112,23 +112,8 @@ update_docker_containers() {
|
||||
# does not include so-elastic-fleet since that container uses so-elastic-agent image
|
||||
local IMAGES_USING_ES_VERSION=("so-elasticsearch")
|
||||
|
||||
rm -rf $SIGNPATH >> "$LOG_FILE" 2>&1
|
||||
mkdir -p $SIGNPATH >> "$LOG_FILE" 2>&1
|
||||
|
||||
# OL10 test path: GnuPG 2.4 enables the keybox daemon (keyboxd) by default, which deadlocks
|
||||
# under the rapid sequential gpg --verify calls below ("waiting for lock ... keydb_search
|
||||
# failed: Connection timed out ... No public key"). Editing the default homedir's common.conf
|
||||
# is unreliable (gpg re-adds use-keyboxd when it re-initializes the homedir), so run all the
|
||||
# image-signature gpg ops in a dedicated homedir whose pre-written common.conf leaves keyboxd
|
||||
# off, forcing the classic keybox. Isolated from the system keyring and deterministic.
|
||||
if [ "$OSVER" = "10" ]; then
|
||||
export GNUPGHOME="$SIGNPATH/gnupg"
|
||||
rm -rf "$GNUPGHOME" >> "$LOG_FILE" 2>&1
|
||||
mkdir -p "$GNUPGHOME" >> "$LOG_FILE" 2>&1
|
||||
chmod 700 "$GNUPGHOME"
|
||||
echo "# keyboxd disabled for SO image signature verification on EL10" > "$GNUPGHOME/common.conf"
|
||||
gpgconf --kill keyboxd gpg-agent >> "$LOG_FILE" 2>&1 || true
|
||||
fi
|
||||
rm -rf $SIGNPATH >> "$LOG_FILE" 2>&1
|
||||
mkdir -p $SIGNPATH >> "$LOG_FILE" 2>&1
|
||||
|
||||
# Let's make sure we have the public key
|
||||
run_check_net_err \
|
||||
|
||||
@@ -165,8 +165,6 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading component template" # false positive (elasticsearch index or template names contain 'error')
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading composable template" # false positive (elasticsearch composable template names contain 'error')
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error while parsing document for index \[.ds-logs-kratos-so-.*object mapping for \[file\]" # false positive (mapping error occuring BEFORE kratos index has rolled over in 2.4.210)
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|No such container" # false positive (telegraf trying to run stats on an old container)
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|passwords do not match" # false positive (automated hydra test)
|
||||
fi
|
||||
|
||||
if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
|
||||
|
||||
@@ -18,18 +18,10 @@ dockergroup:
|
||||
dockerheldpackages:
|
||||
pkg.installed:
|
||||
- pkgs:
|
||||
{% if GLOBALS.os_version|int >= 10 %}
|
||||
# OL10 test path: install latest Docker CE from the public repo (no .el9 builds available)
|
||||
- containerd.io
|
||||
- docker-ce
|
||||
- docker-ce-cli
|
||||
- docker-ce-rootless-extras
|
||||
{% else %}
|
||||
- containerd.io: 2.2.1-1.el9
|
||||
- docker-ce: 3:29.2.1-1.el9
|
||||
- docker-ce-cli: 1:29.2.1-1.el9
|
||||
- docker-ce-rootless-extras: 29.2.1-1.el9
|
||||
{% endif %}
|
||||
- hold: True
|
||||
- update_holds: True
|
||||
|
||||
|
||||
@@ -26,9 +26,7 @@ include:
|
||||
wait_for_elasticsearch_elasticfleet:
|
||||
cmd.run:
|
||||
- name: so-elasticsearch-wait
|
||||
{% endif %}
|
||||
|
||||
{% if GLOBALS.role == "so-fleet" %}
|
||||
# Sync Elastic Agent artifacts to Fleet Node
|
||||
elasticagent_syncartifacts:
|
||||
file.recurse:
|
||||
|
||||
@@ -3958,13 +3958,10 @@ elasticsearch:
|
||||
- vulnerability-mappings
|
||||
- common-settings
|
||||
- common-dynamic-mappings
|
||||
- logs-redis.log@package
|
||||
- logs-redis.log@custom
|
||||
data_stream:
|
||||
allow_custom_routing: false
|
||||
hidden: false
|
||||
ignore_missing_component_templates:
|
||||
- logs-redis.log@custom
|
||||
ignore_missing_component_templates: []
|
||||
index_patterns:
|
||||
- logs-redis.log*
|
||||
priority: 501
|
||||
|
||||
@@ -1,71 +0,0 @@
|
||||
{
|
||||
"description": "zeek.ja4d",
|
||||
"processors": [
|
||||
{
|
||||
"set": {
|
||||
"field": "event.dataset",
|
||||
"value": "ja4d"
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove": {
|
||||
"field": [
|
||||
"host"
|
||||
],
|
||||
"ignore_failure": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"field": "message",
|
||||
"target_field": "message2",
|
||||
"ignore_failure": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename": {
|
||||
"field": "message2.ja4d",
|
||||
"target_field": "hash.ja4d",
|
||||
"ignore_missing": true,
|
||||
"if": "ctx?.message2?.ja4d != null && ctx.message2.ja4d.length() > 0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename": {
|
||||
"field": "message2.client_mac",
|
||||
"target_field": "host.mac",
|
||||
"ignore_missing": true,
|
||||
"if": "ctx?.message2?.client_mac != null && ctx.message2.client_mac.length() > 0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename": {
|
||||
"field": "message2.hostname",
|
||||
"target_field": "host.hostname",
|
||||
"ignore_missing": true,
|
||||
"if": "ctx?.message2?.hostname != null && ctx.message2.hostname.length() > 0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename": {
|
||||
"field": "message2.requested_ip",
|
||||
"target_field": "dhcp.requested_address",
|
||||
"ignore_missing": true,
|
||||
"if": "ctx?.message2?.requested_ip != null && ctx.message2.requested_ip.length() > 0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"rename": {
|
||||
"field": "message2.vendor_class_id",
|
||||
"target_field": "zeek.ja4d.vendor_class_id",
|
||||
"ignore_missing": true,
|
||||
"if": "ctx?.message2?.vendor_class_id != null && ctx.message2.vendor_class_id.length() > 0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"pipeline": {
|
||||
"name": "zeek.common"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -533,23 +533,6 @@ elasticfleet_set_agent_logging_level_warn() {
|
||||
done <<< "$policies_to_update"
|
||||
}
|
||||
|
||||
update_logstash_pipeline_name() {
|
||||
local original_pipeline_name="$1"
|
||||
local new_pipeline_name="$2"
|
||||
|
||||
echo "Checking for conflicting logstash defined_pipelines pillar value."
|
||||
local LOGSTASH_FILE=/opt/so/saltstack/local/pillar/logstash/soc_logstash.sls
|
||||
local MINIONDIR=/opt/so/saltstack/local/pillar/minions
|
||||
for pillar_file in "$LOGSTASH_FILE" "$MINIONDIR"/*.sls; do
|
||||
[[ -f "$pillar_file" ]] || continue
|
||||
if grep -q "$original_pipeline_name$" "$pillar_file"; then
|
||||
echo "Found conflicting defined_pipeline pillar value in $pillar_file. Updating to use the new logstash pipeline name."
|
||||
sed -i "s#$original_pipeline_name\$#$new_pipeline_name#g" "$pillar_file"
|
||||
chown socore:socore "$pillar_file"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
check_transform_health_and_reauthorize() {
|
||||
. /usr/sbin/so-elastic-fleet-common
|
||||
|
||||
@@ -693,10 +676,6 @@ rename_strelka_scan_lnk() {
|
||||
rm -f "$TMP_VALUE_FILE"
|
||||
}
|
||||
|
||||
fix_logstash_0013_lumberjack_pipeline_name() {
|
||||
update_logstash_pipeline_name "so/0013_input_lumberjack_fleet.conf" "so/0013_input_lumberjack_fleet.conf.jinja"
|
||||
}
|
||||
|
||||
up_to_3.1.0() {
|
||||
ensure_postgres_local_pillar
|
||||
ensure_postgres_secret
|
||||
@@ -705,7 +684,6 @@ up_to_3.1.0() {
|
||||
# Clear existing component template state file.
|
||||
rm -f /opt/so/state/esfleet_component_templates.json
|
||||
rename_strelka_scan_lnk
|
||||
fix_logstash_0013_lumberjack_pipeline_name
|
||||
|
||||
INSTALLEDVERSION=3.1.0
|
||||
}
|
||||
@@ -993,9 +971,6 @@ verify_es_version_compatibility() {
|
||||
local is_active_intermediate_upgrade=1
|
||||
# supported upgrade paths for SO-ES versions
|
||||
declare -A es_upgrade_map=(
|
||||
["8.18.4"]="8.18.6 8.18.8 9.0.8"
|
||||
["8.18.6"]="8.18.8 9.0.8"
|
||||
["8.18.8"]="9.0.8"
|
||||
["9.0.8"]="9.3.3"
|
||||
)
|
||||
|
||||
@@ -1019,171 +994,6 @@ verify_es_version_compatibility() {
|
||||
exit 160
|
||||
fi
|
||||
|
||||
compatible_es_versions="$target_es_version"
|
||||
for current_version in "${!es_upgrade_map[@]}"; do
|
||||
# shellcheck disable=SC2076
|
||||
if [[ " ${es_upgrade_map[$current_version]} " =~ " $target_es_version " ]]; then
|
||||
compatible_es_versions+=" $current_version"
|
||||
fi
|
||||
done
|
||||
|
||||
# Check if the given ES version can directly upgrade to the target ES version. Used to assist with catching lagging nodes during the upgrade process
|
||||
es_version_can_upgrade_to_target() {
|
||||
local current_version="$1"
|
||||
# shellcheck disable=SC2076
|
||||
if [[ -n "$current_version" && " $compatible_es_versions " =~ " $current_version " ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
# Gather Elasticsearch cluster version info and verify that each node in the cluster is running a version compatible with the target ES version.
|
||||
verify_searchnodes_es_target_compatibility() {
|
||||
local retries=20
|
||||
local retry_count=0
|
||||
local delay=180
|
||||
local expected_es_nodes searchnode_minions attempt
|
||||
local searchnode_discovery_success=false
|
||||
SEARCHNODE_ES_VERSIONS=""
|
||||
|
||||
for attempt in {1..3}; do
|
||||
if searchnode_minions=$(set -o pipefail; salt-key --out=json --list=accepted 2> /dev/null | jq -r '.minions[]? | select(endswith("searchnode"))'); then
|
||||
searchnode_discovery_success=true
|
||||
break
|
||||
fi
|
||||
|
||||
echo "Failed to retrieve grid searchnodes via salt-key... Retrying in 30 seconds. Attempt $attempt of 3."
|
||||
sleep 30
|
||||
done
|
||||
|
||||
if [[ "$searchnode_discovery_success" != "true" ]]; then
|
||||
echo "Failed to retrieve grid searchnodes via salt-key."
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Always add node running soup to expected es nodes
|
||||
expected_es_nodes="${MINIONID%_*}"
|
||||
while IFS= read -r searchnode_minion; do
|
||||
[[ -z "$searchnode_minion" ]] && continue
|
||||
expected_es_nodes+=$'\n'"${searchnode_minion%_searchnode}"
|
||||
done <<< "$searchnode_minions"
|
||||
|
||||
while [[ $retry_count -lt $retries ]]; do
|
||||
SEARCHNODE_ES_VERSIONS=$(so-elasticsearch-query _nodes/_all/version --retry 5 --retry-delay 10 --fail 2>&1)
|
||||
local exit_status=$?
|
||||
|
||||
if [[ $exit_status -ne 0 ]]; then
|
||||
echo "Failed to retrieve Elasticsearch versions from searchnodes... Retrying in $delay seconds. Attempt $((retry_count + 1)) of $retries."
|
||||
((retry_count++))
|
||||
sleep $delay
|
||||
continue
|
||||
fi
|
||||
|
||||
local all_searchnodes_compatible=true
|
||||
while IFS=$'\t' read -r node current_version; do
|
||||
[[ -z "$node" ]] && continue
|
||||
if ! es_version_can_upgrade_to_target "$current_version"; then
|
||||
echo "Searchnode $node is running Elasticsearch $current_version, which is not directly upgradable to Elasticsearch $target_es_version."
|
||||
all_searchnodes_compatible=false
|
||||
fi
|
||||
done < <(echo "$SEARCHNODE_ES_VERSIONS" | jq -r '.nodes | to_entries[] | [.value.name, .value.version] | @tsv')
|
||||
|
||||
while IFS= read -r expected_es_node; do
|
||||
[[ -z "$expected_es_node" ]] && continue
|
||||
if ! echo "$SEARCHNODE_ES_VERSIONS" | jq -e --arg node "$expected_es_node" '.nodes | to_entries | any(.value.name == $node)' > /dev/null; then
|
||||
echo "Searchnode $expected_es_node did not report an Elasticsearch version. It may be offline or still upgrading."
|
||||
all_searchnodes_compatible=false
|
||||
fi
|
||||
done <<< "$expected_es_nodes"
|
||||
|
||||
if [[ "$all_searchnodes_compatible" == true ]]; then
|
||||
echo "All Searchnodes are upgradable to Elasticsearch $target_es_version."
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "One or more Searchnodes cannot upgrade directly to Elasticsearch $target_es_version. Rechecking in $delay seconds. Attempt $((retry_count + 1)) of $retries."
|
||||
((retry_count++))
|
||||
sleep $delay
|
||||
done
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
# Gather heavynode version info and verify that each node is running a version compatible with the target ES version.
|
||||
verify_heavynodes_es_target_compatibility() {
|
||||
local heavynode_minions attempt
|
||||
local retries=20
|
||||
local retry_count=0
|
||||
local delay=180
|
||||
local heavynode_discovery_success=false
|
||||
HEAVYNODE_ES_VERSIONS=""
|
||||
|
||||
for attempt in {1..3}; do
|
||||
if heavynode_minions=$(set -o pipefail; salt-key --out=json --list=accepted 2> /dev/null | jq -r '.minions[]? | select(endswith("heavynode"))'); then
|
||||
heavynode_discovery_success=true
|
||||
break
|
||||
fi
|
||||
|
||||
echo "Failed to retrieve grid heavynodes via salt-key... Retrying in 30 seconds. Attempt $attempt of 3."
|
||||
sleep 30
|
||||
done
|
||||
|
||||
if [[ "$heavynode_discovery_success" != "true" ]]; then
|
||||
echo "Failed to retrieve grid heavynodes via salt-key."
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ -z "$heavynode_minions" ]]; then
|
||||
echo "No heavynodes detected. Skipping heavynode Elasticsearch version compatibility check."
|
||||
return 0
|
||||
fi
|
||||
|
||||
while [[ $retry_count -lt $retries ]]; do
|
||||
HEAVYNODE_ES_VERSIONS=$(salt -C 'G@role:so-heavynode' cmd.run 'set -o pipefail; so-elasticsearch-query / --retry 5 --retry-delay 10 | jq -er ".version.number"' shell=/bin/bash --out=json 2> /dev/null)
|
||||
local exit_status=$?
|
||||
|
||||
if [[ $exit_status -ne 0 ]]; then
|
||||
echo "Failed to retrieve Elasticsearch version from one or more heavynodes... Retrying in $delay seconds. Attempt $((retry_count + 1)) of $retries."
|
||||
((retry_count++))
|
||||
sleep $delay
|
||||
continue
|
||||
fi
|
||||
|
||||
local all_heavynodes_compatible=true
|
||||
while IFS=$'\t' read -r node current_version; do
|
||||
[[ -z "$node" ]] && continue
|
||||
if ! es_version_can_upgrade_to_target "$current_version"; then
|
||||
echo "Heavynode $node is running Elasticsearch $current_version, which is not directly upgradable to Elasticsearch $target_es_version."
|
||||
all_heavynodes_compatible=false
|
||||
fi
|
||||
done < <(echo "$HEAVYNODE_ES_VERSIONS" | jq -r 'to_entries[] | [.key, .value] | @tsv')
|
||||
|
||||
while IFS= read -r heavynode_minion; do
|
||||
[[ -z "$heavynode_minion" ]] && continue
|
||||
if ! echo "$HEAVYNODE_ES_VERSIONS" | jq -se --arg minion "$heavynode_minion" 'add | has($minion)' > /dev/null; then
|
||||
echo "Heavynode $heavynode_minion did not report an Elasticsearch version. It may be offline or still upgrading."
|
||||
all_heavynodes_compatible=false
|
||||
fi
|
||||
done <<< "$heavynode_minions"
|
||||
|
||||
if [[ "$all_heavynodes_compatible" == true ]]; then
|
||||
echo -e "\nAll heavynodes can upgrade to Elasticsearch $target_es_version."
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "One or more heavynodes cannot upgrade directly to Elasticsearch $target_es_version. Rechecking in $delay seconds. Attempt $((retry_count + 1)) of $retries."
|
||||
((retry_count++))
|
||||
sleep $delay
|
||||
done
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
if [[ ! -f "$es_verification_script" ]]; then
|
||||
create_intermediate_upgrade_verification_script "$es_verification_script"
|
||||
fi
|
||||
|
||||
for statefile in "${es_required_version_statefile_base}"-*; do
|
||||
[[ -f $statefile ]] || continue
|
||||
|
||||
@@ -1202,6 +1012,10 @@ verify_es_version_compatibility() {
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ ! -f "$es_verification_script" ]]; then
|
||||
create_intermediate_upgrade_verification_script "$es_verification_script"
|
||||
fi
|
||||
|
||||
echo -e "\n##############################################################################################################################\n"
|
||||
echo "A previously required intermediate Elasticsearch upgrade was detected. Verifying that all Searchnodes/Heavynodes have successfully upgraded Elasticsearch to $es_required_version_statefile_value before proceeding with soup to avoid potential data loss! This command can take up to an hour to complete."
|
||||
if ! timeout --foreground 4000 bash "$es_verification_script" "$es_required_version_statefile_value" "$statefile"; then
|
||||
@@ -1223,26 +1037,6 @@ verify_es_version_compatibility() {
|
||||
|
||||
# shellcheck disable=SC2076 # Do not want a regex here eg usage " 8.18.8 9.0.8 " =~ " 9.0.8 "
|
||||
if [[ " ${es_upgrade_map[$es_version]} " =~ " $target_es_version " || "$es_version" == "$target_es_version" ]]; then
|
||||
if ! verify_searchnodes_es_target_compatibility || ! verify_heavynodes_es_target_compatibility; then
|
||||
echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
|
||||
|
||||
echo "One or more Searchnode(s)/Heavynode(s) cannot upgrade directly to Elasticsearch $target_es_version. This can happen with soups that include Elasticsearch upgrades being run in quick succession. Typically, this will resolve itself as the grid synchronizes. Please allow time for all Searchnodes/Heavynodes to have upgraded Elasticsearch to a compatible version with $target_es_version before running soup again to avoid potential data loss!"
|
||||
|
||||
if [[ -n "$HEAVYNODE_ES_VERSIONS" ]]; then
|
||||
echo "Current heavynode Elasticsearch versions:"
|
||||
echo "$HEAVYNODE_ES_VERSIONS" | jq '.'
|
||||
fi
|
||||
|
||||
if [[ -n "$SEARCHNODE_ES_VERSIONS" ]]; then
|
||||
echo "Current searchnode Elasticsearch versions:"
|
||||
echo "$SEARCHNODE_ES_VERSIONS" | jq '.nodes | to_entries | map({(.value.name): .value.version}) | sort | add'
|
||||
fi
|
||||
|
||||
echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
|
||||
|
||||
exit 161
|
||||
fi
|
||||
|
||||
# supported upgrade
|
||||
return 0
|
||||
else
|
||||
@@ -1528,13 +1322,7 @@ EOF
|
||||
|
||||
# Keeping this block in case we need to do a hotfix that requires salt update
|
||||
apply_hotfix() {
|
||||
if [[ "$INSTALLEDVERSION" == "3.1.0" ]] ; then
|
||||
# Do not remove this fix_logstash_0013_lumberjack_pipeline_name in future hotfixes without first validating older
|
||||
# installs referencing "so/0013_input_lumberjack_fleet.conf" via pillar are upgradable
|
||||
fix_logstash_0013_lumberjack_pipeline_name
|
||||
else
|
||||
echo "No actions required. ($INSTALLEDVERSION/$HOTFIXVERSION)"
|
||||
fi
|
||||
echo "No actions required. ($INSTALLEDVERSION/$HOTFIXVERSION)"
|
||||
}
|
||||
|
||||
failed_soup_restore_items() {
|
||||
@@ -1606,7 +1394,7 @@ main() {
|
||||
echo "Verifying we have the latest soup script."
|
||||
verify_latest_update_script
|
||||
|
||||
echo "Verifying Elasticsearch version compatibility across the grid before upgrading."
|
||||
echo "Verifying Elasticsearch version compatibility before upgrading."
|
||||
verify_es_version_compatibility
|
||||
|
||||
echo "Let's see if we need to update Security Onion."
|
||||
|
||||
@@ -225,7 +225,6 @@ http {
|
||||
limit_req zone=auth_throttle burst={{ NGINXMERGED.config.throttle_login_burst }} nodelay;
|
||||
limit_req_status 429;
|
||||
proxy_pass http://{{ GLOBALS.manager }}:4433;
|
||||
proxy_set_header Connection "Close";
|
||||
proxy_read_timeout 90;
|
||||
proxy_connect_timeout 90;
|
||||
proxy_set_header Host $host;
|
||||
@@ -238,7 +237,6 @@ http {
|
||||
location ~ ^/auth/.*?(whoami|logout|settings|errors|webauthn.js) {
|
||||
rewrite /auth/(.*) /$1 break;
|
||||
proxy_pass http://{{ GLOBALS.manager }}:4433;
|
||||
proxy_set_header Connection "Close";
|
||||
proxy_read_timeout 90;
|
||||
proxy_connect_timeout 90;
|
||||
proxy_set_header Host $host;
|
||||
|
||||
@@ -46,10 +46,10 @@ postgresinitdir:
|
||||
- require:
|
||||
- file: postgresconfdir
|
||||
|
||||
postgresinitdb:
|
||||
postgresinitusers:
|
||||
file.managed:
|
||||
- name: /opt/so/conf/postgres/init/init-db.sh
|
||||
- source: salt://postgres/files/init-db.sh
|
||||
- name: /opt/so/conf/postgres/init/init-users.sh
|
||||
- source: salt://postgres/files/init-users.sh
|
||||
- user: 939
|
||||
- group: 939
|
||||
- mode: 755
|
||||
|
||||
@@ -31,7 +31,7 @@ so-postgres:
|
||||
- POSTGRES_DB=securityonion
|
||||
# Passwords are delivered via mounted 0600 secret files, not plaintext env vars.
|
||||
# The upstream postgres image resolves POSTGRES_PASSWORD_FILE; entrypoint.sh and
|
||||
# init-db.sh resolve SO_POSTGRES_PASS_FILE the same way.
|
||||
# init-users.sh resolve SO_POSTGRES_PASS_FILE the same way.
|
||||
- POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password
|
||||
- SO_POSTGRES_USER={{ SO_POSTGRES_USER }}
|
||||
- SO_POSTGRES_PASS_FILE=/run/secrets/so_postgres_pass
|
||||
@@ -46,7 +46,7 @@ so-postgres:
|
||||
- /opt/so/conf/postgres/postgresql.conf:/conf/postgresql.conf:ro
|
||||
- /opt/so/conf/postgres/pg_hba.conf:/conf/pg_hba.conf:ro
|
||||
- /opt/so/conf/postgres/secrets:/run/secrets:ro
|
||||
- /opt/so/conf/postgres/init/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh:ro
|
||||
- /opt/so/conf/postgres/init/init-users.sh:/docker-entrypoint-initdb.d/init-users.sh:ro
|
||||
- /etc/pki/postgres.crt:/conf/postgres.crt:ro
|
||||
- /etc/pki/postgres.key:/conf/postgres.key:ro
|
||||
- /etc/pki/tls/certs/intca.crt:/conf/ca.crt:ro
|
||||
@@ -70,7 +70,7 @@ so-postgres:
|
||||
- watch:
|
||||
- file: postgresconf
|
||||
- file: postgreshba
|
||||
- file: postgresinitdb
|
||||
- file: postgresinitusers
|
||||
- file: postgres_super_secret
|
||||
- file: postgres_app_secret
|
||||
- x509: postgres_crt
|
||||
@@ -78,7 +78,7 @@ so-postgres:
|
||||
- require:
|
||||
- file: postgresconf
|
||||
- file: postgreshba
|
||||
- file: postgresinitdb
|
||||
- file: postgresinitusers
|
||||
- file: postgres_super_secret
|
||||
- file: postgres_app_secret
|
||||
- x509: postgres_crt
|
||||
|
||||
@@ -39,15 +39,17 @@ postgres_wait_ready:
|
||||
- require:
|
||||
- docker_container: so-postgres
|
||||
|
||||
# Ensure the shared Telegraf database exists. init-db.sh only runs on a
|
||||
# Ensure the shared Telegraf database exists. init-users.sh only runs on a
|
||||
# fresh data dir, so hosts upgraded onto an existing /nsm/postgres volume
|
||||
# would otherwise never get so_telegraf.
|
||||
postgres_create_telegraf_db:
|
||||
cmd.run:
|
||||
- name: /usr/sbin/so-telegraf-postgres create_db
|
||||
- name: |
|
||||
if ! docker exec so-postgres psql -U postgres -tAc "SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1; then
|
||||
docker exec so-postgres psql -v ON_ERROR_STOP=1 -U postgres -c "CREATE DATABASE so_telegraf"
|
||||
fi
|
||||
- require:
|
||||
- cmd: postgres_wait_ready
|
||||
- file: postgres_sbin
|
||||
|
||||
# Provision the shared group role and schema once. Every per-minion role is a
|
||||
# member of so_telegraf, and each Telegraf connection does SET ROLE so_telegraf
|
||||
@@ -55,26 +57,68 @@ postgres_create_telegraf_db:
|
||||
# on first write are owned by the group role and every member can INSERT/SELECT.
|
||||
postgres_telegraf_group_role:
|
||||
cmd.run:
|
||||
- name: /usr/sbin/so-telegraf-postgres group_role
|
||||
- name: |
|
||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'so_telegraf') THEN
|
||||
CREATE ROLE so_telegraf NOLOGIN;
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
GRANT CONNECT ON DATABASE so_telegraf TO so_telegraf;
|
||||
CREATE SCHEMA IF NOT EXISTS telegraf AUTHORIZATION so_telegraf;
|
||||
GRANT USAGE, CREATE ON SCHEMA telegraf TO so_telegraf;
|
||||
CREATE SCHEMA IF NOT EXISTS partman;
|
||||
CREATE EXTENSION IF NOT EXISTS pg_partman SCHEMA partman;
|
||||
CREATE EXTENSION IF NOT EXISTS pg_cron;
|
||||
-- Telegraf (running as so_telegraf) calls partman.create_parent()
|
||||
-- on first write of each metric, which needs USAGE on the partman
|
||||
-- schema, EXECUTE on its functions/procedures, and write access to
|
||||
-- partman.part_config so it can register new partitioned parents.
|
||||
GRANT USAGE, CREATE ON SCHEMA partman TO so_telegraf;
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA partman TO so_telegraf;
|
||||
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA partman TO so_telegraf;
|
||||
GRANT EXECUTE ON ALL PROCEDURES IN SCHEMA partman TO so_telegraf;
|
||||
-- partman creates per-parent template tables (partman.template_*) at
|
||||
-- runtime; default privileges extend DML/sequence access to them.
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA partman
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO so_telegraf;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA partman
|
||||
GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO so_telegraf;
|
||||
-- Hourly partman maintenance. cron.schedule is idempotent by jobname.
|
||||
SELECT cron.schedule(
|
||||
'telegraf-partman-maintenance',
|
||||
'17 * * * *',
|
||||
'CALL partman.run_maintenance_proc()'
|
||||
);
|
||||
EOSQL
|
||||
- require:
|
||||
- cmd: postgres_create_telegraf_db
|
||||
- file: postgres_sbin
|
||||
|
||||
{% set creds = salt['pillar.get']('telegraf:postgres_creds', {}) %}
|
||||
{% for mid, entry in creds.items() %}
|
||||
{% if entry.get('user') and entry.get('pass') %}
|
||||
{% set u = entry.user %}
|
||||
{% set p = entry.pass %}
|
||||
{% set p = entry.pass | replace("'", "''") %}
|
||||
|
||||
postgres_telegraf_role_{{ u }}:
|
||||
cmd.run:
|
||||
- name: /usr/sbin/so-telegraf-postgres user
|
||||
- env:
|
||||
- ROLE_USER: {{ u | tojson }}
|
||||
- ROLE_PASS: {{ p | tojson }}
|
||||
- hide_output: True
|
||||
- name: |
|
||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '{{ u }}') THEN
|
||||
EXECUTE format('CREATE ROLE %I WITH LOGIN PASSWORD %L', '{{ u }}', '{{ p }}');
|
||||
ELSE
|
||||
EXECUTE format('ALTER ROLE %I WITH PASSWORD %L', '{{ u }}', '{{ p }}');
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
GRANT CONNECT ON DATABASE so_telegraf TO "{{ u }}";
|
||||
GRANT so_telegraf TO "{{ u }}";
|
||||
EOSQL
|
||||
- require:
|
||||
- file: postgres_sbin
|
||||
- cmd: postgres_telegraf_group_role
|
||||
|
||||
{% endif %}
|
||||
@@ -86,12 +130,21 @@ postgres_telegraf_role_{{ u }}:
|
||||
{% set retention = salt['pillar.get']('postgres:telegraf:retention_days', 14) | int %}
|
||||
postgres_telegraf_retention_reconcile:
|
||||
cmd.run:
|
||||
- name: /usr/sbin/so-telegraf-postgres retention
|
||||
- env:
|
||||
- RETENTION_DAYS: {{ retention }}
|
||||
- name: |
|
||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'pg_partman') THEN
|
||||
UPDATE partman.part_config
|
||||
SET retention = '{{ retention }} days',
|
||||
retention_keep_table = false
|
||||
WHERE parent_table LIKE 'telegraf.%';
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
EOSQL
|
||||
- require:
|
||||
- cmd: postgres_telegraf_group_role
|
||||
- file: postgres_sbin
|
||||
|
||||
{% endif %}
|
||||
|
||||
|
||||
@@ -7,29 +7,15 @@
|
||||
|
||||
. /usr/sbin/so-common
|
||||
|
||||
# Without pipefail, a pipeline's exit status is gzip's. A failed pg_dumpall would
|
||||
# otherwise be masked by a successful gzip, silently producing a valid .gz that
|
||||
# holds a truncated dump.
|
||||
set -o pipefail
|
||||
|
||||
# Backups contain role password hashes and full chat data; keep them 0600.
|
||||
umask 0077
|
||||
|
||||
TODAY=$(date '+%Y_%m_%d')
|
||||
BACKUPDIR=/nsm/backup
|
||||
BACKUPFILE="$BACKUPDIR/so-postgres-backup-$TODAY.sql.gz"
|
||||
TMPFILE="$BACKUPFILE.tmp"
|
||||
MAXBACKUPS=7
|
||||
LOGFILE=/opt/so/log/postgres/backup.log
|
||||
|
||||
log() {
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') $*" >> "$LOGFILE"
|
||||
}
|
||||
|
||||
mkdir -p "$BACKUPDIR"
|
||||
|
||||
# Remove any temp files left behind by a previously crashed run
|
||||
rm -f "$BACKUPDIR"/so-postgres-backup-*.sql.gz.tmp
|
||||
mkdir -p $BACKUPDIR
|
||||
|
||||
# Skip if already backed up today
|
||||
if [ -f "$BACKUPFILE" ]; then
|
||||
@@ -41,33 +27,13 @@ if ! docker ps --format '{{.Names}}' | grep -q '^so-postgres$'; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Always clean up the temp file on exit; the success path clears this trap
|
||||
# after the atomic rename so the finished backup is not deleted.
|
||||
trap 'rm -f "$TMPFILE"' EXIT
|
||||
# Dump all databases and roles, compress
|
||||
docker exec so-postgres pg_dumpall -U postgres | gzip > "$BACKUPFILE"
|
||||
|
||||
# Dump all databases and roles, compress. Write to a temp file so the final
|
||||
# filename only ever appears for a complete, verified backup.
|
||||
if ! docker exec so-postgres pg_dumpall -U postgres | gzip > "$TMPFILE"; then
|
||||
log "ERROR: pg_dumpall/gzip failed; backup aborted"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Verify the compressed stream is intact before publishing it
|
||||
if ! gzip -t "$TMPFILE"; then
|
||||
log "ERROR: backup failed gzip integrity check; backup aborted"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Atomically publish the verified backup
|
||||
mv "$TMPFILE" "$BACKUPFILE"
|
||||
trap - EXIT
|
||||
log "OK: wrote $BACKUPFILE"
|
||||
|
||||
# Retention cleanup (only reached after a successful backup). The glob is
|
||||
# restricted to finished backups so an in-progress .tmp can never be counted.
|
||||
NUMBACKUPS=$(find "$BACKUPDIR" -type f -name "so-postgres-backup-*.sql.gz" | wc -l)
|
||||
# Retention cleanup
|
||||
NUMBACKUPS=$(find $BACKUPDIR -type f -name "so-postgres-backup*" | wc -l)
|
||||
while [ "$NUMBACKUPS" -gt "$MAXBACKUPS" ]; do
|
||||
OLDEST=$(find "$BACKUPDIR" -type f -name "so-postgres-backup-*.sql.gz" -printf '%T+ %p\n' | sort | head -n 1 | awk -F" " '{print $2}')
|
||||
OLDEST=$(find $BACKUPDIR -type f -name "so-postgres-backup*" -printf '%T+ %p\n' | sort | head -n 1 | awk -F" " '{print $2}')
|
||||
rm -f "$OLDEST"
|
||||
NUMBACKUPS=$(find "$BACKUPDIR" -type f -name "so-postgres-backup-*.sql.gz" | wc -l)
|
||||
NUMBACKUPS=$(find $BACKUPDIR -type f -name "so-postgres-backup*" | wc -l)
|
||||
done
|
||||
|
||||
@@ -1,110 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Provision Telegraf state inside the so-postgres container.
|
||||
# Usage: so-telegraf-postgres <subcommand>
|
||||
# create_db Ensure the so_telegraf database exists.
|
||||
# group_role Provision the so_telegraf group role, telegraf/partman schemas,
|
||||
# pg_partman, pg_cron, and the hourly partman maintenance job.
|
||||
# user Create or update a per-minion login role granted to so_telegraf.
|
||||
# Env: ROLE_USER, ROLE_PASS.
|
||||
# retention Reconcile partman retention on telegraf parents.
|
||||
# Env: RETENTION_DAYS.
|
||||
|
||||
cmd="${1:?subcommand required}"
|
||||
|
||||
case "$cmd" in
|
||||
create_db)
|
||||
if ! docker exec so-postgres psql -U postgres -tAc \
|
||||
"SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1; then
|
||||
docker exec so-postgres psql -v ON_ERROR_STOP=1 -U postgres \
|
||||
-c "CREATE DATABASE so_telegraf"
|
||||
fi
|
||||
;;
|
||||
|
||||
group_role)
|
||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'so_telegraf') THEN
|
||||
CREATE ROLE so_telegraf NOLOGIN;
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
GRANT CONNECT ON DATABASE so_telegraf TO so_telegraf;
|
||||
CREATE SCHEMA IF NOT EXISTS telegraf AUTHORIZATION so_telegraf;
|
||||
GRANT USAGE, CREATE ON SCHEMA telegraf TO so_telegraf;
|
||||
CREATE SCHEMA IF NOT EXISTS partman;
|
||||
CREATE EXTENSION IF NOT EXISTS pg_partman SCHEMA partman;
|
||||
CREATE EXTENSION IF NOT EXISTS pg_cron;
|
||||
-- Telegraf (running as so_telegraf) calls partman.create_parent()
|
||||
-- on first write of each metric, which needs USAGE on the partman
|
||||
-- schema, EXECUTE on its functions/procedures, and write access to
|
||||
-- partman.part_config so it can register new partitioned parents.
|
||||
GRANT USAGE, CREATE ON SCHEMA partman TO so_telegraf;
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA partman TO so_telegraf;
|
||||
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA partman TO so_telegraf;
|
||||
GRANT EXECUTE ON ALL PROCEDURES IN SCHEMA partman TO so_telegraf;
|
||||
-- partman creates per-parent template tables (partman.template_*) at
|
||||
-- runtime; default privileges extend DML/sequence access to them.
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA partman
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO so_telegraf;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA partman
|
||||
GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO so_telegraf;
|
||||
-- Hourly partman maintenance. cron.schedule is idempotent by jobname.
|
||||
SELECT cron.schedule(
|
||||
'telegraf-partman-maintenance',
|
||||
'17 * * * *',
|
||||
'CALL partman.run_maintenance_proc()'
|
||||
);
|
||||
EOSQL
|
||||
;;
|
||||
|
||||
user)
|
||||
: "${ROLE_USER:?ROLE_USER is required}"
|
||||
: "${ROLE_PASS:?ROLE_PASS is required}"
|
||||
# psql does not substitute :vars inside dollar-quoted strings, so the
|
||||
# conditional CREATE/ALTER is built outside any DO block and dispatched
|
||||
# with \gexec. format() handles identifier/literal quoting.
|
||||
docker exec -i so-postgres psql \
|
||||
-v ON_ERROR_STOP=1 \
|
||||
-v role_user="$ROLE_USER" \
|
||||
-v role_pass="$ROLE_PASS" \
|
||||
-U postgres -d so_telegraf <<'EOSQL'
|
||||
SELECT format(
|
||||
CASE WHEN EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'role_user')
|
||||
THEN 'ALTER ROLE %I WITH LOGIN PASSWORD %L'
|
||||
ELSE 'CREATE ROLE %I WITH LOGIN PASSWORD %L'
|
||||
END,
|
||||
:'role_user',
|
||||
:'role_pass'
|
||||
) \gexec
|
||||
GRANT CONNECT ON DATABASE so_telegraf TO :"role_user";
|
||||
GRANT so_telegraf TO :"role_user";
|
||||
EOSQL
|
||||
;;
|
||||
|
||||
retention)
|
||||
: "${RETENTION_DAYS:?RETENTION_DAYS is required}"
|
||||
# \gset + \if guards against a missing pg_partman without using a DO
|
||||
# block (psql :var substitution doesn't reach into dollar-quoted code).
|
||||
docker exec -i so-postgres psql \
|
||||
-v ON_ERROR_STOP=1 \
|
||||
-v retention_days="$RETENTION_DAYS" \
|
||||
-U postgres -d so_telegraf <<'EOSQL'
|
||||
SELECT CASE WHEN EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'pg_partman')
|
||||
THEN 'true' ELSE 'false' END AS has_partman \gset
|
||||
\if :has_partman
|
||||
UPDATE partman.part_config
|
||||
SET retention = :'retention_days' || ' days',
|
||||
retention_keep_table = false
|
||||
WHERE parent_table LIKE 'telegraf.%';
|
||||
\endif
|
||||
EOSQL
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "Unknown subcommand: $cmd" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
@@ -1,6 +1,5 @@
|
||||
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
||||
{# OL10 test path uses public repos; skip the SO repo state (which removes public repos and points at /nsm/repo) #}
|
||||
{% if GLOBALS.os == 'OEL' and GLOBALS.os_version|int == 9 %}
|
||||
{% if GLOBALS.os == 'OEL' %}
|
||||
include:
|
||||
- repo.client.oracle
|
||||
{% endif %}
|
||||
@@ -25,11 +25,15 @@ soc:
|
||||
target: ''
|
||||
links:
|
||||
- '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}" OR "{:network.community_id}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source.as.organization.name source.geo.country_name | groupby destination.as.organization.name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid&gridId={gridId}'
|
||||
- '/#/hunt?q=("{:log.id.fuid}" OR {:log.id.uids|any} OR "{:network.community_id}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source.as.organization.name source.geo.country_name | groupby destination.as.organization.name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid&gridId={gridId}'
|
||||
- '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source.as.organization.name source.geo.country_name | groupby destination.as.organization.name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid&gridId={gridId}'
|
||||
- '/#/hunt?q=("{:log.id.fuid}" OR {:log.id.uids|any}) | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source.as.organization.name source.geo.country_name | groupby destination.as.organization.name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid&gridId={gridId}'
|
||||
- '/#/hunt?q=("{:log.id.fuid}" OR "{:network.community_id}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source.as.organization.name source.geo.country_name | groupby destination.as.organization.name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid&gridId={gridId}'
|
||||
- '/#/hunt?q=("{:log.id.uid}" OR "{:network.community_id}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source.as.organization.name source.geo.country_name | groupby destination.as.organization.name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid&gridId={gridId}'
|
||||
- '/#/hunt?q=({:log.id.uids|any} OR "{:network.community_id}") | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source.as.organization.name source.geo.country_name | groupby destination.as.organization.name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid&gridId={gridId}'
|
||||
- '/#/hunt?q="{:log.id.fuid}" | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source.as.organization.name source.geo.country_name | groupby destination.as.organization.name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid&gridId={gridId}'
|
||||
- '/#/hunt?q="{:log.id.uid}" | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source.as.organization.name source.geo.country_name | groupby destination.as.organization.name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid&gridId={gridId}'
|
||||
- '/#/hunt?q={:log.id.uids|any} | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source.as.organization.name source.geo.country_name | groupby destination.as.organization.name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid&gridId={gridId}'
|
||||
- '/#/hunt?q="{:network.community_id}" | groupby event.module* | groupby -sankey event.module* event.dataset | groupby event.dataset | groupby source.ip source.port destination.ip destination.port | groupby network.protocol | groupby source.as.organization.name source.geo.country_name | groupby destination.as.organization.name destination.geo.country_name | groupby rule.name rule.category event.severity_label | groupby dns.query.name | groupby file.mime_type | groupby http.virtual_host http.uri | groupby notice.note notice.message notice.sub_message | groupby ssl.server_name | groupby source.ip host.hostname user.name event.action event.type process.executable process.pid&gridId={gridId}'
|
||||
- name: actionPcap
|
||||
description: actionPcapHelp
|
||||
@@ -156,7 +160,7 @@ soc:
|
||||
- host.domain
|
||||
- host.hostname
|
||||
- dhcp.message_types
|
||||
- log.id.uid
|
||||
- log.id.uids
|
||||
'::dnp3':
|
||||
- soc_timestamp
|
||||
- event.dataset
|
||||
|
||||
@@ -261,7 +261,7 @@ strelka:
|
||||
priority: 5
|
||||
options:
|
||||
limit: 1000
|
||||
'ScanLnk':
|
||||
'ScanLNK':
|
||||
- positive:
|
||||
flavors:
|
||||
- 'lnk_file'
|
||||
|
||||
@@ -99,7 +99,7 @@ strelka:
|
||||
'ScanJpeg': *scannerOptions
|
||||
'ScanJson': *scannerOptions
|
||||
'ScanLibarchive': *scannerOptions
|
||||
'ScanLnk': *scannerOptions
|
||||
'ScanLNK': *scannerOptions
|
||||
'ScanLsb': *scannerOptions
|
||||
'ScanLzma': *scannerOptions
|
||||
'ScanMacho': *scannerOptions
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
telegraf:
|
||||
enabled: False
|
||||
output: INFLUXDB
|
||||
output: BOTH
|
||||
config:
|
||||
interval: '30s'
|
||||
metric_batch_size: 1000
|
||||
|
||||
+2
-27
@@ -119,7 +119,7 @@ base:
|
||||
- kafka
|
||||
- pcap.cleanup
|
||||
|
||||
'*_manager and G@saltversion:{{saltversion}} and not I@node_data:False':
|
||||
'*_manager or *_managerhype and G@saltversion:{{saltversion}} and not I@node_data:False':
|
||||
- match: compound
|
||||
- salt.master
|
||||
- registry
|
||||
@@ -146,32 +146,6 @@ base:
|
||||
- stig
|
||||
- kafka
|
||||
|
||||
'*_managerhype and G@saltversion:{{saltversion}} and not I@node_data:False':
|
||||
- match: compound
|
||||
- salt.master
|
||||
- registry
|
||||
- nginx
|
||||
- influxdb
|
||||
- postgres
|
||||
- strelka.manager
|
||||
- soc
|
||||
- kratos
|
||||
- hydra
|
||||
- firewall
|
||||
- manager
|
||||
- sensoroni
|
||||
- telegraf
|
||||
- backup.config_backup
|
||||
- elasticsearch
|
||||
- logstash
|
||||
- redis
|
||||
- elastic-fleet-package-registry
|
||||
- kibana
|
||||
- elastalert
|
||||
- utility
|
||||
- elasticfleet
|
||||
- kafka
|
||||
|
||||
'*_managerhype and I@features:vrt and G@saltversion:{{saltversion}}':
|
||||
- match: compound
|
||||
- manager.hypervisor
|
||||
@@ -312,6 +286,7 @@ base:
|
||||
- libvirt
|
||||
- libvirt.images
|
||||
- elasticfleet.install_agent_grid
|
||||
- stig
|
||||
|
||||
'*_desktop and G@saltversion:{{saltversion}}':
|
||||
- sensoroni
|
||||
|
||||
@@ -31,7 +31,6 @@
|
||||
'so_model': INIT.GRAINS.get('sosmodel',''),
|
||||
'sensoroni_key': INIT.PILLAR.sensoroni.config.sensoronikey,
|
||||
'os': INIT.GRAINS.os,
|
||||
'os_version': INIT.GRAINS.osmajorrelease,
|
||||
'os_family': INIT.GRAINS.os_family,
|
||||
'application_urls': {},
|
||||
'manager_roles': [
|
||||
|
||||
+5
-21
@@ -903,14 +903,14 @@ detect_cloud() {
|
||||
|
||||
detect_os() {
|
||||
title "Detecting Base OS"
|
||||
if [ -f /etc/oracle-release ] && grep -qE "release (9|10)\b" /etc/oracle-release; then
|
||||
if [ -f /etc/redhat-release ] && grep -q "Red Hat Enterprise Linux release 9" /etc/redhat-release && [ -f /etc/oracle-release ]; then
|
||||
OS=oracle
|
||||
OSVER=$(grep -oE "release [0-9]+" /etc/oracle-release | grep -oE "[0-9]+")
|
||||
OSVER=9
|
||||
is_oracle=true
|
||||
is_rpm=true
|
||||
is_supported=true
|
||||
else
|
||||
info "This OS is not supported. Security Onion requires Oracle Linux 9 or 10."
|
||||
info "This OS is not supported. Security Onion requires Oracle Linux 9."
|
||||
fail_setup
|
||||
fi
|
||||
|
||||
@@ -1783,15 +1783,6 @@ ensure_pyyaml() {
|
||||
# - securityonion/salt/salt/minion.defaults.yaml
|
||||
|
||||
securityonion_repo() {
|
||||
if [[ "$OSVER" == "10" ]]; then
|
||||
# TEST PATH: Oracle Linux 10 uses the public OL10 + EPEL + Docker CE repos.
|
||||
# Keep the stock /etc/yum.repos.d/* in place, skip the SO mirror and local reposync.
|
||||
gpg_rpm_import
|
||||
logCmd "dnf -y install oracle-epel-release-el10"
|
||||
logCmd "dnf -y config-manager --add-repo https://download.docker.com/linux/rhel/docker-ce.repo"
|
||||
logCmd "dnf repolist"
|
||||
return
|
||||
fi
|
||||
# Remove all the current repos
|
||||
logCmd "dnf -v clean all"
|
||||
logCmd "mkdir -vp /root/oldrepos"
|
||||
@@ -1886,19 +1877,12 @@ saltify() {
|
||||
info "Installing Salt $SALTVERSION"
|
||||
chmod u+x ../salt/salt/scripts/bootstrap-salt.sh
|
||||
|
||||
# Normally Salt packages come from the SO mirror, so -r disables the bootstrap's own repo setup.
|
||||
# On the OL10 test path there is no SO mirror, so let bootstrap configure the public Salt repo.
|
||||
local saltrepoflag="-r"
|
||||
if [[ "$OSVER" == "10" ]]; then
|
||||
saltrepoflag=""
|
||||
fi
|
||||
|
||||
if [[ $waitforstate ]]; then
|
||||
# install all for a manager
|
||||
retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh $saltrepoflag -M -X stable $SALTVERSION" || fail_setup
|
||||
retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -M -X stable $SALTVERSION" || fail_setup
|
||||
else
|
||||
# just a minion
|
||||
retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh $saltrepoflag -X stable $SALTVERSION" || fail_setup
|
||||
retry 30 10 "bash ../salt/salt/scripts/bootstrap-salt.sh -r -X stable $SALTVERSION" || fail_setup
|
||||
fi
|
||||
|
||||
salt_install_module_deps
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Reference in New Issue
Block a user