CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-17 22:42:51 +01:00
Code Issues Packages Projects Releases Wiki Activity

merge with 2.4.120, fix merge conflicts

Browse Source
This commit is contained in:
m0duspwnens
2024-10-25 15:09:38 -04:00
parent 0476585370 a76a2d8e9f
commit feb700393e
520 changed files with 12109 additions and 250 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
salt/common/init.sls
View File

@@ -182,6 +182,7 @@ sostatus_log:
file.managed:
- name: /opt/so/log/sostatus/status.log
- mode: 644
- replace: False
# Install sostatus check cron. This is used to populate Grid.
so-status_check_cron:

74
salt/common/tools/sbin/so-common
View File

@@ -8,12 +8,6 @@
# Elastic agent is not managed by salt. Because of this we must store this base information in a
# script that accompanies the soup system. Since so-common is one of those special soup files,
# and since this same logic is required during installation, it's included in this file.
ELASTIC_AGENT_TARBALL_VERSION="8.14.3"
ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
ELASTIC_AGENT_MD5="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
ELASTIC_AGENT_EXPANSION_DIR=/nsm/elastic-fleet/artifacts/beats/elastic-agent
DEFAULT_SALT_DIR=/opt/so/saltstack/default
DOC_BASE_URL="https://docs.securityonion.net/en/2.4"
@@ -174,6 +168,46 @@ check_salt_minion_status() {
return $status
}
# Compare es versions and return the highest version
compare_es_versions() {
# Save the original IFS
local OLD_IFS="$IFS"
IFS=.
local i ver1=($1) ver2=($2)
# Restore the original IFS
IFS="$OLD_IFS"
# Determine the maximum length between the two version arrays
local max_len=${#ver1[@]}
if [[ ${#ver2[@]} -gt $max_len ]]; then
max_len=${#ver2[@]}
fi
# Compare each segment of the versions
for ((i=0; i<max_len; i++)); do
# If a segment in ver1 or ver2 is missing, set it to 0
if [[ -z ${ver1[i]} ]]; then
ver1[i]=0
fi
if [[ -z ${ver2[i]} ]]; then
ver2[i]=0
fi
if ((10#${ver1[i]} > 10#${ver2[i]})); then
echo "$1"
return 0
fi
if ((10#${ver1[i]} < 10#${ver2[i]})); then
echo "$2"
return 0
fi
done
echo "$1" # If versions are equal, return either
return 0
}
copy_new_files() {
# Copy new files over to the salt dir
cd $UPDATE_DIR
@@ -263,11 +297,6 @@ fail() {
exit 1
}
get_random_value() {
length=${1:-20}
head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1
}
get_agent_count() {
if [ -f /opt/so/log/agents/agentstatus.log ]; then
AGENTCOUNT=$(cat /opt/so/log/agents/agentstatus.log | grep -wF active | awk '{print $2}')
@@ -276,6 +305,27 @@ get_agent_count() {
fi
}
get_elastic_agent_vars() {
local path="${1:-/opt/so/saltstack/default}"
local defaultsfile="${path}/salt/elasticsearch/defaults.yaml"
if [ -f "$defaultsfile" ]; then
ELASTIC_AGENT_TARBALL_VERSION=$(egrep " +version: " $defaultsfile | awk -F: '{print $2}' | tr -d '[:space:]')
ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
ELASTIC_AGENT_MD5="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
ELASTIC_AGENT_EXPANSION_DIR=/nsm/elastic-fleet/artifacts/beats/elastic-agent
else
fail "Could not find salt/elasticsearch/defaults.yaml"
fi
}
get_random_value() {
length=${1:-20}
head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1
}
gpg_rpm_import() {
if [[ $is_oracle ]]; then
if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then
@@ -627,6 +677,8 @@ has_uppercase() {
}
update_elastic_agent() {
local path="${1:-/opt/so/saltstack/default}"
get_elastic_agent_vars "$path"
echo "Checking if Elastic Agent update is necessary..."
download_and_verify "$ELASTIC_AGENT_URL" "$ELASTIC_AGENT_MD5_URL" "$ELASTIC_AGENT_FILE" "$ELASTIC_AGENT_MD5" "$ELASTIC_AGENT_EXPANSION_DIR"
}

29
salt/common/tools/sbin/so-image-common
View File

@@ -112,6 +112,10 @@ update_docker_containers() {
container_list
fi
# all the images using ELASTICSEARCHDEFAULTS.elasticsearch.version
# does not include so-elastic-fleet since that container uses so-elastic-agent image
local IMAGES_USING_ES_VERSION=("so-elasticsearch")
rm -rf $SIGNPATH >> "$LOG_FILE" 2>&1
mkdir -p $SIGNPATH >> "$LOG_FILE" 2>&1
@@ -139,15 +143,36 @@ update_docker_containers() {
$PROGRESS_CALLBACK $i
fi
if [[ " ${IMAGES_USING_ES_VERSION[*]} " =~ [[:space:]]${i}[[:space:]] ]]; then
# this is an es container so use version defined in elasticsearch defaults.yaml
local UPDATE_DIR='/tmp/sogh/securityonion'
if [ ! -d "$UPDATE_DIR" ]; then
UPDATE_DIR=/securityonion
fi
local v1=0
local v2=0
if [[ -f "$UPDATE_DIR/salt/elasticsearch/defaults.yaml" ]]; then
v1=$(egrep " +version: " "$UPDATE_DIR/salt/elasticsearch/defaults.yaml" | awk -F: '{print $2}' | tr -d '[:space:]')
fi
if [[ -f "$DEFAULT_SALT_DIR/salt/elasticsearch/defaults.yaml" ]]; then
v2=$(egrep " +version: " "$DEFAULT_SALT_DIR/salt/elasticsearch/defaults.yaml" | awk -F: '{print $2}' | tr -d '[:space:]')
fi
local highest_es_version=$(compare_es_versions "$v1" "$v2")
local image=$i:$highest_es_version$IMAGE_TAG_SUFFIX
local sig_url=https://sigs.securityonion.net/es-$highest_es_version/$image.sig
else
# this is not an es container so use the so version for the version
local image=$i:$VERSION$IMAGE_TAG_SUFFIX
local sig_url=https://sigs.securityonion.net/$VERSION/$image.sig
fi
# Pull down the trusted docker image
local image=$i:$VERSION$IMAGE_TAG_SUFFIX
run_check_net_err \
"docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" \
"Could not pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1
# Get signature
run_check_net_err \
"curl --retry 5 --retry-delay 60 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" \
"curl --retry 5 --retry-delay 60 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' $sig_url --output $SIGNPATH/$image.sig" \
"Could not pull signature file for $image, please ensure connectivity to https://sigs.securityonion.net " \
noretry >> "$LOG_FILE" 2>&1
# Dump our hash values

5
salt/common/tools/sbin/so-log-check
View File

@@ -95,6 +95,8 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|shutdown process" # server not yet ready (logstash waiting on elastic)
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|contain valid certificates" # server not yet ready (logstash waiting on elastic)
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failedaction" # server not yet ready (logstash waiting on elastic)
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|block in start_workers" # server not yet ready (logstash waiting on elastic)
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|block in buffer_initialize" # server not yet ready (logstash waiting on elastic)
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no route to host" # server not yet ready
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|not running" # server not yet ready
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|unavailable" # server not yet ready
@@ -147,6 +149,7 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status 200" # false positive (request successful, contained error string in content)
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|app_layer.error" # false positive (suricata 7) in stats.log e.g. app_layer.error.imap.parser | Total | 0
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|is not an ip string literal" # false positive (Open Canary logging out blank IP addresses)
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncing rule" # false positive (rule sync log line includes rule name which can contain 'error')
fi
if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
@@ -170,6 +173,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|cannot join on an empty table" # InfluxDB flux query, import nodes
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|exhausting result iterator" # InfluxDB flux query mismatched table results (temporary data issue)
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to finish run" # InfluxDB rare error, self-recoverable
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unable to gather disk name" # InfluxDB known error, can't read disks because the container doesn't have them mounted
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration"
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets"
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|use of closed"
@@ -205,6 +209,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|detect-parse" # Suricata encountering a malformed rule
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|integrity check failed" # Detections: Exclude false positive due to automated testing
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncErrors" # Detections: Not an actual error
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Initialized license manager" # SOC log: before fields.status was changed to fields.licenseStatus
fi
RESULT=0

67
salt/common/tools/sbin_jinja/so-raid-status
View File

@@ -9,6 +9,9 @@
. /usr/sbin/so-common
software_raid=("SOSMN" "SOSMN-DE02" "SOSSNNV" "SOSSNNV-DE02" "SOS10k-DE02" "SOS10KNV" "SOS10KNV-DE02" "SOS10KNV-DE02" "SOS2000-DE02" "SOS-GOFAST-LT-DE02" "SOS-GOFAST-MD-DE02" "SOS-GOFAST-HV-DE02")
hardware_raid=("SOS1000" "SOS1000F" "SOSSN7200" "SOS5000" "SOS4000")
{%- if salt['grains.get']('sosmodel', '') %}
{%- set model = salt['grains.get']('sosmodel') %}
model={{ model }}
@@ -16,33 +19,42 @@ model={{ model }}
if [[ $model =~ ^(SO2AMI01|SO2AZI01|SO2GCI01)$ ]]; then
exit 0
fi
for i in "${software_raid[@]}"; do
if [[ "$model" == $i ]]; then
is_softwareraid=true
is_hwraid=false
break
fi
done
for i in "${hardware_raid[@]}"; do
if [[ "$model" == $i ]]; then
is_softwareraid=false
is_hwraid=true
break
fi
done
{%- else %}
echo "This is not an appliance"
exit 0
{%- endif %}
if [[ $model =~ ^(SOS10K|SOS500|SOS1000|SOS1000F|SOS4000|SOSSN7200|SOSSNNV|SOSMN)$ ]]; then
is_bossraid=true
fi
if [[ $model =~ ^(SOSSNNV|SOSMN)$ ]]; then
is_swraid=true
fi
if [[ $model =~ ^(SOS10K|SOS500|SOS1000|SOS1000F|SOS4000|SOSSN7200)$ ]]; then
is_hwraid=true
fi
check_nsm_raid() {
PERCCLI=$(/opt/raidtools/perccli/perccli64 /c0/v0 show|grep RAID|grep Optl)
MEGACTL=$(/opt/raidtools/megasasctl |grep optimal)
if [[ $APPLIANCE == '1' ]]; then
if [[ "$model" == "SOS500" || "$model" == "SOS500-DE02" ]]; then
#This doesn't have raid
HWRAID=0
else
if [[ -n $PERCCLI ]]; then
HWRAID=0
elif [[ -n $MEGACTL ]]; then
HWRAID=0
else
HWRAID=1
fi
fi
fi
}
@@ -50,17 +62,27 @@ check_nsm_raid() {
check_boss_raid() {
MVCLI=$(/usr/local/bin/mvcli info -o vd |grep status |grep functional)
MVTEST=$(/usr/local/bin/mvcli info -o vd | grep "No adapter")
BOSSNVMECLI=$(/usr/local/bin/mnv_cli info -o vd -i 0 | grep Functional)
# Check to see if this is a SM based system
if [[ -z $MVTEST ]]; then
if [[ -n $MVCLI ]]; then
# Is this NVMe Boss Raid?
if [[ "$model" =~ "-DE02" ]]; then
if [[ -n $BOSSNVMECLI ]]; then
BOSSRAID=0
else
BOSSRAID=1
fi
else
# This doesn't have boss raid so lets make it 0
BOSSRAID=0
# Check to see if this is a SM based system
if [[ -z $MVTEST ]]; then
if [[ -n $MVCLI ]]; then
BOSSRAID=0
else
BOSSRAID=1
fi
else
# This doesn't have boss raid so lets make it 0
BOSSRAID=0
fi
fi
}
@@ -79,14 +101,13 @@ SWRAID=0
BOSSRAID=0
HWRAID=0
if [[ $is_hwraid ]]; then
if [[ "$is_hwraid" == "true" ]]; then
check_nsm_raid
check_boss_raid
fi
if [[ $is_bossraid ]]; then
check_boss_raid
fi
if [[ $is_swraid ]]; then
if [[ "$is_softwareraid" == "true" ]]; then
check_software_raid
check_boss_raid
fi
sum=$(($SWRAID + $BOSSRAID + $HWRAID))