CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-06 17:22:49 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into foxtrot

Browse Source 
# Conflicts:
#	salt/common/tools/sbin/soup
This commit is contained in:
William Wernert
2021-03-17 10:23:51 -04:00
parent 129db23062 fbdb627ab7
commit c0163108ab
13 changed files with 198 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
pillar/logstash/manager.sls
View File

@@ -3,7 +3,6 @@ logstash:
pipelines:
manager:
config:
- so/0008_input_fleet_livequery.conf.jinja
- so/0009_input_beats.conf
- so/0010_input_hhbeats.conf
- so/9999_output_redis.conf.jinja

1
pillar/logstash/search.sls
View File

@@ -8,7 +8,6 @@ logstash:
- so/9002_output_import.conf.jinja
- so/9034_output_syslog.conf.jinja
- so/9100_output_osquery.conf.jinja
- so/9101_output_osquery_livequery.conf.jinja
- so/9400_output_suricata.conf.jinja
- so/9500_output_beats.conf.jinja
- so/9600_output_ossec.conf.jinja

59
salt/common/tools/sbin/so-common
View File

@@ -86,6 +86,19 @@ add_interface_bond0() {
fi
}
check_airgap() {
# See if this is an airgap install
AIRGAP=$(cat /opt/so/saltstack/local/pillar/global.sls | grep airgap: | awk '{print $2}')
if [[ "$AIRGAP" == "True" ]]; then
is_airgap=0
UPDATE_DIR=/tmp/soagupdate/SecurityOnion
AGDOCKER=/tmp/soagupdate/docker
AGREPO=/tmp/soagupdate/Packages
else
is_airgap=1
fi
}
check_container() {
docker ps | grep "$1:" > /dev/null 2>&1
return $?
@@ -97,6 +110,46 @@ check_password() {
return $?
}
check_elastic_license() {
[ -n "$TESTING" ] && return
# See if the user has already accepted the license
if [ ! -f /opt/so/state/yeselastic.txt ]; then
elastic_license
else
echo "Elastic License has already been accepted"
fi
}
elastic_license() {
read -r -d '' message <<- EOM
\n
Starting in Elastic Stack version 7.11, the Elastic Stack binaries are only available under the Elastic License:
https://securityonion.net/elastic-license
Please review the Elastic License:
https://www.elastic.co/licensing/elastic-license
Do you agree to the terms of the Elastic License?
If so, type AGREE to accept the Elastic License and continue. Otherwise, press Enter to exit this program without making any changes.
EOM
AGREED=$(whiptail --title "Security Onion Setup" --inputbox \
"$message" 20 75 3>&1 1>&2 2>&3)
if [ "$AGREED" = 'AGREE' ]; then
mkdir -p /opt/so/state
touch /opt/so/state/yeselastic.txt
else
echo "Starting in 2.3.40 you must accept the Elastic license if you want to run Security Onion."
exit 1
fi
}
fail() {
msg=$1
echo "ERROR: $msg"
@@ -250,6 +303,12 @@ set_minionid() {
MINIONID=$(lookup_grain id)
}
set_palette() {
if [ "$OS" == ubuntu ]; then
update-alternatives --set newt-palette /etc/newt/palette.original
fi
}
set_version() {
CURRENTVERSION=0.0.0
if [ -f /etc/soversion ]; then

103
salt/common/tools/sbin/soup
View File

@@ -19,13 +19,12 @@
UPDATE_DIR=/tmp/sogh/securityonion
INSTALLEDVERSION=$(cat /etc/soversion)
POSTVERSION=$INSTALLEDVERSION
INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk {'print $2'})
DEFAULT_SALT_DIR=/opt/so/saltstack/default
BATCHSIZE=5
SOUP_LOG=/root/soup.log
exec 3>&1 1>${SOUP_LOG} 2>&1
add_common() {
cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
@@ -101,19 +100,6 @@ update_registry() {
salt-call state.apply registry queue=True
}
check_airgap() {
# See if this is an airgap install
AIRGAP=$(cat /opt/so/saltstack/local/pillar/global.sls | grep airgap: | awk '{print $2}')
if [[ "$AIRGAP" == "True" ]]; then
is_airgap=0
UPDATE_DIR=/tmp/soagupdate/SecurityOnion
AGDOCKER=/tmp/soagupdate/docker
AGREPO=/tmp/soagupdate/Packages
else
is_airgap=1
fi
}
check_sudoers() {
if grep -q "so-setup" /etc/sudoers; then
echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"."
@@ -243,21 +229,9 @@ masterunlock() {
fi
}
playbook() {
echo "Applying playbook settings"
if [[ "$INSTALLEDVERSION" =~ rc.1 ]]; then
salt-call state.apply playbook.OLD_db_init
rm -f /opt/so/rules/elastalert/playbook/*.yaml
so-playbook-ruleupdate >> /root/soup_playbook_rule_update.log 2>&1 &
fi
if [[ "$INSTALLEDVERSION" != 2.3.30 ]]; then
so-playbook-sigma-refresh >> /root/soup_playbook_sigma_refresh.log 2>&1 &
fi
}
pillar_changes() {
preupgrade_changes() {
# This function is to add any new pillar items if needed.
echo "Checking to see if pillar changes are needed."
echo "Checking to see if changes are needed."
[[ "$INSTALLEDVERSION" =~ rc.1 ]] && rc1_to_rc2
[[ "$INSTALLEDVERSION" =~ rc.2 ]] && rc2_to_rc3
@@ -266,6 +240,26 @@ pillar_changes() {
[[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30
}
postupgrade_changes() {
# This function is to add any new pillar items if needed.
echo "Running post upgrade processes."
[[ "$POSTVERSION" =~ rc.1 ]] && post_rc1_to_rc2
[[ "$POSTVERSION" == 2.3.20 || "$POSTVERSION" == 2.3.21 ]] && post_2.3.2X_to_2.3.30
}
post_rc1_to_2.3.21() {
salt-call state.apply playbook.OLD_db_init
rm -f /opt/so/rules/elastalert/playbook/*.yaml
so-playbook-ruleupdate >> /root/soup_playbook_rule_update.log 2>&1 &
POSTVERSION=2.3.21
}
post_2.3.2X_to_2.3.30() {
so-playbook-sigma-refresh >> /root/soup_playbook_sigma_refresh.log 2>&1 &
POSTVERSION=2.3.30
}
rc1_to_rc2() {
# Move the static file to global.sls
@@ -408,16 +402,26 @@ up_2.3.2X_to_2.3.30() {
check_log_size_limit
}
space_check() {
# Check to see if there is enough space
verify_upgradespace() {
CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//')
if [ "$CURRENTSPACE" -lt "10" ]; then
echo "You are low on disk space. Upgrade will try and clean up space.";
clean_dockers
echo "You are low on disk space."
return 1
else
echo "Plenty of space for upgrading"
return 0
fi
}
upgrade_space() {
if ! verify_upgradespace; then
clean_dockers
if ! verify_upgradespace; then
echo "There is not enough space to perform the upgrade. Please free up space and try again"
exit 1
fi
else
echo "You have enough space for upgrade. Proceeding with soup."
fi
}
thehive_maint() {
@@ -546,6 +550,7 @@ verify_latest_update_script() {
}
main () {
echo "### Preparing soup at `date` ###"
while getopts ":b" opt; do
case "$opt" in
b ) # process option b
@@ -572,6 +577,8 @@ check_airgap
echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
echo ""
set_os
set_palette
check_elastic_license
echo ""
if [ $is_airgap -eq 0 ]; then
# Let's mount the ISO since this is airgap
@@ -598,7 +605,7 @@ fi
echo "Let's see if we need to update Security Onion."
upgrade_check
space_check
upgrade_space
echo "Checking for Salt Master and Minion updates."
upgrade_check_salt
@@ -648,8 +655,7 @@ else
echo ""
fi
echo "Making pillar changes."
pillar_changes
preupgrade_changes
echo ""
if [ $is_airgap -eq 0 ]; then
@@ -703,7 +709,7 @@ echo "Starting Salt Master service."
systemctl start salt-master
echo "Running a highstate. This could take several minutes."
salt-call state.highstate -l info queue=True
playbook
postupgrade_changes
unmount_update
thehive_maint
@@ -740,6 +746,7 @@ NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l)
if [ $NUM_MINIONS -gt 1 ]; then
cat << EOF
This appears to be a distributed deployment. Other nodes should update themselves at the next Salt highstate (typically within 15 minutes). Do not manually restart anything until you know that all the search/heavy nodes in your deployment are updated. This is especially important if you are using true clustering for Elasticsearch.
Each minion is on a random 15 minute check-in period and things like network bandwidth can be a factor in how long the actual upgrade takes. If you have a heavy node on a slow link, it is going to take a while to get the containers to it. Depending on what changes happened between the versions, Elasticsearch might not be able to talk to said heavy node until the update is complete.
@@ -750,7 +757,23 @@ For more information, please see https://docs.securityonion.net/en/2.3/soup.html
EOF
fi
echo "### soup has been served at `date` ###"
}
main "$@" | tee /dev/fd/3
cat << EOF
SOUP - Security Onion UPdater
Please review the following for more information about the update process and recent updates:
https://docs.securityonion.net/soup
https://blog.securityonion.net
Please note that soup only updates Security Onion components and does NOT update the underlying operating system (OS). When you installed Security Onion, there was an option to automatically update the OS packages. If you did not enable this option, then you will want to ensure that the OS is fully updated before running soup.
Press Enter to continue or Ctrl-C to cancel.
EOF
read input
main "$@" | tee -a $SOUP_LOG

2
salt/elasticsearch/files/ingest/common
View File

@@ -32,8 +32,6 @@
{ "rename": { "field": "category", "target_field": "event.category", "ignore_failure": true, "ignore_missing": true } },
{ "rename": { "field": "message2.community_id", "target_field": "network.community_id", "ignore_failure": true, "ignore_missing": true } },
{ "lowercase": { "field": "event.dataset", "ignore_failure": true, "ignore_missing": true } },
{ "convert": { "field": "destination.port", "type": "integer", "ignore_failure": true, "ignore_missing": true } },
{ "convert": { "field": "source.port", "type": "integer", "ignore_failure": true, "ignore_missing": true } },
{ "convert": { "field": "log.id.uid", "type": "string", "ignore_failure": true, "ignore_missing": true } },
{ "convert": { "field": "agent.id", "type": "string", "ignore_failure": true, "ignore_missing": true } },
{ "convert": { "field": "event.severity", "type": "integer", "ignore_failure": true, "ignore_missing": true } },

19
salt/elasticsearch/templates/so/so-common-template.json
View File

@@ -51,16 +51,29 @@
"match_mapping_type": "string",
"path_match": "*.ip",
"mapping": {
"type": "ip"
"type": "ip",
"fields" : {
"keyword" : {
"ignore_above" : 45,
"type" : "keyword"
}
}
}
}
},
{
"port": {
"match_mapping_type": "string",
"path_match": "*.port",
"mapping": {
"type": "integer"
"type": "integer",
"fields" : {
"keyword" : {
"ignore_above" : 6,
"type" : "keyword"
}
}
}
}
},

41
salt/firewall/init.sls
View File

@@ -26,15 +26,6 @@ iptables_fix_fwd:
- position: 1
- target: DOCKER-USER
# Allow related/established sessions
iptables_allow_established:
iptables.append:
- table: filter
- chain: INPUT
- jump: ACCEPT
- match: conntrack
- ctstate: 'RELATED,ESTABLISHED'
# I like pings
iptables_allow_pings:
iptables.append:
@@ -77,17 +68,6 @@ enable_docker_user_fw_policy:
- out-interface: docker0
- position: 1
enable_docker_user_established:
iptables.insert:
- table: filter
- chain: DOCKER-USER
- jump: ACCEPT
- in-interface: '!docker0'
- out-interface: docker0
- position: 1
- match: conntrack
- ctstate: 'RELATED,ESTABLISHED'
{% set count = namespace(value=0) %}
{% for chain, hg in assigned_hostgroups.chain.items() %}
{% for hostgroup, portgroups in assigned_hostgroups.chain[chain].hostgroups.items() %}
@@ -120,6 +100,27 @@ enable_docker_user_established:
{% endfor %}
{% endfor %}
# Allow related/established sessions
iptables_allow_established:
iptables.insert:
- table: filter
- chain: INPUT
- jump: ACCEPT
- position: 1
- match: conntrack
- ctstate: 'RELATED,ESTABLISHED'
enable_docker_user_established:
iptables.insert:
- table: filter
- chain: DOCKER-USER
- jump: ACCEPT
- in-interface: '!docker0'
- out-interface: docker0
- position: 1
- match: conntrack
- ctstate: 'RELATED,ESTABLISHED'
# Block icmp timestamp response
block_icmp_timestamp_reply:
iptables.append:

6
salt/kibana/bin/so-kibana-config-load
View File

@@ -24,3 +24,9 @@ THECOOKIE=$(curl -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}
# Load saved objects
curl -b "sid=$THECOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson > /dev/null 2>&1
# Disable certain Features from showing up in the Kibana UI
echo
echo "Setting up default Space:"
curl -b "sid=$THECOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["enterpriseSearch","siem","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","fleet"]} ' | jq

3
salt/strelka/files/filestream/filestream.yaml
View File

@@ -19,7 +19,8 @@ files:
- '/nsm/strelka/unprocessed/*'
delete: false
gatekeeper: true
processed: '/nsm/strelka/processed'
response:
report: 5s
delta: 5s
staging: '/nsm/strelka/processed'
staging: '/nsm/strelka/staging'

9
salt/strelka/init.sls
View File

@@ -86,6 +86,13 @@ strelkaprocessed:
- group: 939
- makedirs: True
strelkastaging:
file.directory:
- name: /nsm/strelka/staging
- user: 939
- group: 939
- makedirs: True
strelkaunprocessed:
file.directory:
- name: /nsm/strelka/unprocessed
@@ -96,7 +103,7 @@ strelkaunprocessed:
# Check to see if Strelka frontend port is available
strelkaportavailable:
cmd.run:
- name: netstat -utanp | grep ":57314" | grep -qv docker && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314. Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0
- name: netstat -utanp | grep ":57314" | grep -qvE 'docker|TIME_WAIT' && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314. Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0
strelka_coordinator:
docker_container.running:

5
salt/telegraf/etc/telegraf.conf
View File

@@ -618,11 +618,8 @@
# # Read stats from one or more Elasticsearch servers or clusters
{% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone'] %}
[[inputs.elasticsearch]]
# ## specify a list of one or more Elasticsearch servers
# # you can add username and password to your url to use basic authentication:
# # servers = ["http://user:pass@localhost:9200"]
servers = ["https://{{ MANAGER }}:9200"]
insecure_skip_verify = true
{% elif grains['role'] in ['so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode'] %}
[[inputs.elasticsearch]]
servers = ["https://{{ NODEIP }}:9200"]

13
setup/so-functions
View File

@@ -1243,8 +1243,13 @@ es_heapsize() {
# https://www.elastic.co/guide/en/elasticsearch/guide/current/heap-sizing.html
ES_HEAP_SIZE="25000m"
else
# Set heap size to 25% of available memory
ES_HEAP_SIZE=$(( total_mem / 4 ))"m"
# Set heap size to 33% of available memory
ES_HEAP_SIZE=$(( total_mem / 3 ))
if [ "$ES_HEAP_SIZE" -ge 25001 ] ; then
ES_HEAP_SIZE="25000m"
else
ES_HEAP_SIZE=$ES_HEAP_SIZE"m"
fi
fi
export ES_HEAP_SIZE
@@ -1686,8 +1691,8 @@ manager_global() {
" so-zeek:"\
" shards: 5"\
" warm: 7"\
" close: 365"\
" delete: 45"\
" close: 45"\
" delete: 365"\
"minio:"\
" access_key: '$ACCESS_KEY'"\
" access_secret: '$ACCESS_SECRET'"\

8
setup/so-setup
View File

@@ -168,10 +168,8 @@ local_sbin="$(pwd)/../salt/common/tools/sbin"
export PATH=$PATH:$local_sbin
set_network_dev_status_list
set_palette >> $setup_log 2>&1
if [ "$OS" == ubuntu ]; then
update-alternatives --set newt-palette /etc/newt/palette.original >> $setup_log 2>&1
fi
# Kernel messages can overwrite whiptail screen #812
# https://github.com/Security-Onion-Solutions/securityonion/issues/812
@@ -274,6 +272,10 @@ if [[ ( $is_manager || $is_import ) && $is_iso ]]; then
fi
fi
if [[ $is_manager || $is_import ]]; then
check_elastic_license
fi
if ! [[ -f $install_opt_file ]]; then
if [[ $is_manager && $is_sensor ]]; then
check_requirements "standalone"