CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-06 17:22:49 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5 from Security-Onion-Solutions/dev

Browse Source 
Dev Sync
This commit is contained in:
Masaya-A
2021-03-23 14:27:29 +09:00
committed by GitHub
parent 236373cda2 197693df4e
commit 151376a18f
43 changed files with 1755 additions and 1190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@@ -1,6 +1,6 @@
## Security Onion 2.3.30
## Security Onion 2.3.40
Security Onion 2.3.30 is here!
Security Onion 2.3.40 is here!
## Screenshots

22
VERIFY_ISO.md
View File

@@ -1,16 +1,16 @@
### 2.3.30 ISO image built on 2021/03/01
### 2.3.40 ISO image built on 2021/03/22
### Download and Verify
2.3.30 ISO image:
https://download.securityonion.net/file/securityonion/securityonion-2.3.30.iso
2.3.40 ISO image:
https://download.securityonion.net/file/securityonion/securityonion-2.3.40.iso
MD5: 65202BA0F7661A5E27087F097B8E571E
SHA1: 14E842E39EDBB55A104263281CF25BF88A2E9D67
SHA256: 210B37B9E3DFC827AFE2940E2C87B175ADA968EDD04298A5926F63D9269847B7
MD5: FB72C0675F262A714B287BB33CE82504
SHA1: E8F5A9AA23990DF794611F9A178D88414F5DA81C
SHA256: DB125D6E770F75C3FD35ABE3F8A8B21454B7A7618C2B446D11B6AC8574601070
Signature for ISO image:
https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.30.iso.sig
https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.40.iso.sig
Signing key:
https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS
@@ -24,22 +24,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/ma
Download the signature file for the ISO:
```
wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.30.iso.sig
wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.40.iso.sig
```
Download the ISO image:
```
wget https://download.securityonion.net/file/securityonion/securityonion-2.3.30.iso
wget https://download.securityonion.net/file/securityonion/securityonion-2.3.40.iso
```
Verify the downloaded ISO image using the signature file:
```
gpg --verify securityonion-2.3.30.iso.sig securityonion-2.3.30.iso
gpg --verify securityonion-2.3.40.iso.sig securityonion-2.3.40.iso
```
The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
```
gpg: Signature made Mon 01 Mar 2021 02:15:28 PM EST using RSA key ID FE507013
gpg: Signature made Mon 22 Mar 2021 09:35:50 AM EDT using RSA key ID FE507013
gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
gpg: WARNING: This key is not certified with a trusted signature!
gpg: There is no indication that the signature belongs to the owner.

2
VERSION
View File

@@ -1 +1 @@
2.3.40
2.3.50

1
pillar/logstash/manager.sls
View File

@@ -3,7 +3,6 @@ logstash:
pipelines:
manager:
config:
- so/0008_input_fleet_livequery.conf.jinja
- so/0009_input_beats.conf
- so/0010_input_hhbeats.conf
- so/9999_output_redis.conf.jinja

1
pillar/logstash/search.sls
View File

@@ -8,7 +8,6 @@ logstash:
- so/9002_output_import.conf.jinja
- so/9034_output_syslog.conf.jinja
- so/9100_output_osquery.conf.jinja
- so/9101_output_osquery_livequery.conf.jinja
- so/9400_output_suricata.conf.jinja
- so/9500_output_beats.conf.jinja
- so/9600_output_ossec.conf.jinja

15
salt/common/init.sls
View File

@@ -90,7 +90,6 @@ commonpkgs:
- ntpdate
- jq
- python3-docker
- docker-ce
- curl
- ca-certificates
- software-properties-common
@@ -104,12 +103,15 @@ commonpkgs:
- python3-dateutil
- python3-m2crypto
- python3-mysqldb
- python3-packaging
- git
heldpackages:
pkg.installed:
- pkgs:
- containerd.io: 1.2.13-2
- docker-ce: 5:19.03.14~3-0~ubuntu-bionic
- containerd.io: 1.4.4-1
- docker-ce: 5:20.10.5~3-0~ubuntu-bionic
- docker-ce-cli: 5:20.10.5~3-0~ubuntu-bionic
- docker-ce-rootless-extras: 5:20.10.5~3-0~ubuntu-bionic
- hold: True
- update_holds: True
@@ -135,6 +137,7 @@ commonpkgs:
- python36-dateutil
- python36-m2crypto
- python36-mysql
- python36-packaging
- yum-utils
- device-mapper-persistent-data
- lvm2
@@ -144,8 +147,10 @@ commonpkgs:
heldpackages:
pkg.installed:
- pkgs:
- containerd.io: 1.2.13-3.2.el7
- docker-ce: 3:19.03.14-3.el7
- containerd.io: 1.4.4-3.1.el7
- docker-ce: 3:20.10.5-3.el7
- docker-ce-cli: 1:20.10.5-3.el7
- docker-ce-rootless-extras: 20.10.5-3.el7
- hold: True
- update_holds: True
{% endif %}

79
salt/common/tools/sbin/so-common
View File

@@ -86,6 +86,19 @@ add_interface_bond0() {
fi
}
check_airgap() {
# See if this is an airgap install
AIRGAP=$(cat /opt/so/saltstack/local/pillar/global.sls | grep airgap: | awk '{print $2}')
if [[ "$AIRGAP" == "True" ]]; then
is_airgap=0
UPDATE_DIR=/tmp/soagupdate/SecurityOnion
AGDOCKER=/tmp/soagupdate/docker
AGREPO=/tmp/soagupdate/Packages
else
is_airgap=1
fi
}
check_container() {
docker ps | grep "$1:" > /dev/null 2>&1
return $?
@@ -97,6 +110,46 @@ check_password() {
return $?
}
check_elastic_license() {
[ -n "$TESTING" ] && return
# See if the user has already accepted the license
if [ ! -f /opt/so/state/yeselastic.txt ]; then
elastic_license
else
echo "Elastic License has already been accepted"
fi
}
elastic_license() {
read -r -d '' message <<- EOM
\n
Starting in Elastic Stack version 7.11, the Elastic Stack binaries are only available under the Elastic License:
https://securityonion.net/elastic-license
Please review the Elastic License:
https://www.elastic.co/licensing/elastic-license
Do you agree to the terms of the Elastic License?
If so, type AGREE to accept the Elastic License and continue. Otherwise, press Enter to exit this program without making any changes.
EOM
AGREED=$(whiptail --title "Security Onion Setup" --inputbox \
"$message" 20 75 3>&1 1>&2 2>&3)
if [ "${AGREED^^}" = 'AGREE' ]; then
mkdir -p /opt/so/state
touch /opt/so/state/yeselastic.txt
else
echo "Starting in 2.3.40 you must accept the Elastic license if you want to run Security Onion."
exit 1
fi
}
fail() {
msg=$1
echo "ERROR: $msg"
@@ -250,6 +303,12 @@ set_minionid() {
MINIONID=$(lookup_grain id)
}
set_palette() {
if [ "$OS" == ubuntu ]; then
update-alternatives --set newt-palette /etc/newt/palette.original
fi
}
set_version() {
CURRENTVERSION=0.0.0
if [ -f /etc/soversion ]; then
@@ -340,6 +399,26 @@ valid_int() {
# {% raw %}
valid_proxy() {
local proxy=$1
local url_prefixes=( 'http://' 'https://' )
local has_prefix=false
for prefix in "${url_prefixes[@]}"; do
echo "$proxy" | grep -q "$prefix" && has_prefix=true && proxy=${proxy#"$prefix"} && break
done
local url_arr
mapfile -t url_arr <<< "$(echo "$proxy" | tr ":" "\n")"
local valid_url=true
if ! valid_ip4 "${url_arr[0]}" && ! valid_fqdn "${url_arr[0]}" && ! valid_hostname "${url_arr[0]}"; then
valid_url=false
fi
[[ $has_prefix == true ]] && [[ $valid_url == true ]] && return 0 || return 1
}
valid_string() {
local str=$1
local min_length=${2:-1}

2
salt/common/tools/sbin/so-cortex-user-add
View File

@@ -30,7 +30,7 @@ fi
USER=$1
CORTEX_KEY=$(lookup_pillar cortexkey)
CORTEX_KEY=$(lookup_pillar cortexorguserkey)
CORTEX_API_URL="$(lookup_pillar url_base)/cortex/api"
CORTEX_ORG_NAME=$(lookup_pillar cortexorgname)
CORTEX_USER=$USER

2
salt/common/tools/sbin/so-cortex-user-enable
View File

@@ -30,7 +30,7 @@ fi
USER=$1
CORTEX_KEY=$(lookup_pillar cortexkey)
CORTEX_KEY=$(lookup_pillar cortexorguserkey)
CORTEX_API_URL="$(lookup_pillar url_base)/cortex/api"
CORTEX_USER=$USER

85
salt/common/tools/sbin/so-docker-prune Executable file
View File

@@ -0,0 +1,85 @@
#!/usr/bin/env python3
# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import sys, argparse, re, docker
from packaging.version import Version, InvalidVersion
from itertools import groupby, chain
def get_image_name(string) -> str:
return ':'.join(string.split(':')[:-1])
def get_so_image_basename(string) -> str:
return get_image_name(string).split('/so-')[-1]
def get_image_version(string) -> str:
ver = string.split(':')[-1]
if ver == 'latest':
# Version doesn't like "latest", so use a high semver
return '999999.9.9'
else:
try:
Version(ver)
except InvalidVersion:
# Strip the last substring following a hyphen for automated branches
ver = '-'.join(ver.split('-')[:-1])
return ver
def main(quiet):
client = docker.from_env()
image_list = client.images.list(filters={ 'dangling': False })
# Map list of image objects to flattened list of tags (format: "name:version")
tag_list = list(chain.from_iterable(list(map(lambda x: x.attrs.get('RepoTags'), image_list))))
# Filter to only SO images (base name begins with "so-")
tag_list = list(filter(lambda x: re.match(r'^.*\/so-[^\/]*$', get_image_name(x)), tag_list))
# Group tags into lists by base name (sort by same projection first)
tag_list.sort(key=lambda x: get_so_image_basename(x))
grouped_tag_lists = [ list(it) for _, it in groupby(tag_list, lambda x: get_so_image_basename(x)) ]
no_prunable = True
for t_list in grouped_tag_lists:
try:
# Keep the 2 most current images
t_list.sort(key=lambda x: Version(get_image_version(x)), reverse=True)
if len(t_list) <= 2:
continue
else:
no_prunable = False
for tag in t_list[2:]:
if not quiet: print(f'Removing image {tag}')
client.images.remove(tag)
except InvalidVersion as e:
print(f'so-{get_so_image_basename(t_list[0])}: {e.args[0]}', file=sys.stderr)
exit(1)
if no_prunable and not quiet:
print('No Security Onion images to prune')
if __name__ == "__main__":
main_parser = argparse.ArgumentParser(add_help=False)
main_parser.add_argument('-q', '--quiet', action='store_const', const=True, required=False)
args = main_parser.parse_args(sys.argv[1:])
main(args.quiet)

13
salt/common/tools/sbin/so-kibana-space-defaults Normal file
View File

@@ -0,0 +1,13 @@
. /usr/sbin/so-common
wait_for_web_response "http://localhost:5601/app/kibana" "Elastic"
## This hackery will be removed if using Elastic Auth ##
# Let's snag a cookie from Kibana
THECOOKIE=$(curl -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
# Disable certain Features from showing up in the Kibana UI
echo
echo "Setting up default Space:"
curl -b "sid=$THECOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","siem","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","fleet"]} ' >> /opt/so/log/kibana/misc.log
echo

0
salt/common/tools/sbin/so-monitor-add Normal file → Executable file
View File

0
salt/common/tools/sbin/so-playbook-sigma-refresh Normal file → Executable file
View File

0
salt/common/tools/sbin/so-raid-status Normal file → Executable file
View File

25
salt/common/tools/sbin/so-rule Normal file → Executable file
View File

@@ -37,11 +37,9 @@ def print_err(string: str):
def check_apply(args: dict, prompt: bool = True):
cmd_arr = ['salt-call', 'state.apply', 'idstools', 'queue=True']
if args.apply:
print('Configuration updated. Applying idstools state...')
return subprocess.run(cmd_arr)
print('Configuration updated. Applying changes:')
return apply()
else:
if prompt:
message = 'Configuration updated. Would you like to apply your changes now? (y/N) '
@@ -51,12 +49,24 @@ def check_apply(args: dict, prompt: bool = True):
if answer.lower() in [ 'n', '' ]:
return 0
else:
print('Applying idstools state...')
return subprocess.run(cmd_arr)
print('Applying changes:')
return apply()
else:
return 0
def apply():
salt_cmd = ['salt-call', 'state.apply', '-l', 'quiet', 'idstools.sync_files', 'queue=True']
update_cmd = ['so-rule-update']
print('Syncing config files...')
cmd = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL)
if cmd.returncode == 0:
print('Updating rules...')
return subprocess.run(update_cmd).returncode
else:
return cmd.returncode
def find_minion_pillar() -> str:
regex = '^.*_(manager|managersearch|standalone|import|eval)\.sls$'
@@ -442,9 +452,6 @@ def main():
modify.print_help()
sys.exit(0)
if isinstance(exit_code, subprocess.CompletedProcess):
sys.exit(exit_code.returncode)
else:
sys.exit(exit_code)

0
salt/common/tools/sbin/so-suricata-testrule Normal file → Executable file
View File

116
salt/common/tools/sbin/soup
View File

@@ -19,13 +19,12 @@
UPDATE_DIR=/tmp/sogh/securityonion
INSTALLEDVERSION=$(cat /etc/soversion)
POSTVERSION=$INSTALLEDVERSION
INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk {'print $2'})
DEFAULT_SALT_DIR=/opt/so/saltstack/default
BATCHSIZE=5
SOUP_LOG=/root/soup.log
exec 3>&1 1>${SOUP_LOG} 2>&1
add_common() {
cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
@@ -101,19 +100,6 @@ update_registry() {
salt-call state.apply registry queue=True
}
check_airgap() {
# See if this is an airgap install
AIRGAP=$(cat /opt/so/saltstack/local/pillar/global.sls | grep airgap: | awk '{print $2}')
if [[ "$AIRGAP" == "True" ]]; then
is_airgap=0
UPDATE_DIR=/tmp/soagupdate/SecurityOnion
AGDOCKER=/tmp/soagupdate/docker
AGREPO=/tmp/soagupdate/Packages
else
is_airgap=1
fi
}
check_sudoers() {
if grep -q "so-setup" /etc/sudoers; then
echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"."
@@ -243,21 +229,9 @@ masterunlock() {
fi
}
playbook() {
echo "Applying playbook settings"
if [[ "$INSTALLEDVERSION" =~ rc.1 ]]; then
salt-call state.apply playbook.OLD_db_init
rm -f /opt/so/rules/elastalert/playbook/*.yaml
so-playbook-ruleupdate >> /root/soup_playbook_rule_update.log 2>&1 &
fi
if [[ "$INSTALLEDVERSION" != 2.3.30 ]]; then
so-playbook-sigma-refresh >> /root/soup_playbook_sigma_refresh.log 2>&1 &
fi
}
pillar_changes() {
preupgrade_changes() {
# This function is to add any new pillar items if needed.
echo "Checking to see if pillar changes are needed."
echo "Checking to see if changes are needed."
[[ "$INSTALLEDVERSION" =~ rc.1 ]] && rc1_to_rc2
[[ "$INSTALLEDVERSION" =~ rc.2 ]] && rc2_to_rc3
@@ -266,6 +240,34 @@ pillar_changes() {
[[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30
}
postupgrade_changes() {
# This function is to add any new pillar items if needed.
echo "Running post upgrade processes."
[[ "$POSTVERSION" =~ rc.1 ]] && post_rc1_to_rc2
[[ "$POSTVERSION" == 2.3.20 || "$POSTVERSION" == 2.3.21 ]] && post_2.3.2X_to_2.3.30
[[ "$POSTVERSION" == 2.3.30 ]] && post_2.3.30_to_2.3.40
}
post_rc1_to_2.3.21() {
salt-call state.apply playbook.OLD_db_init
rm -f /opt/so/rules/elastalert/playbook/*.yaml
so-playbook-ruleupdate >> /root/soup_playbook_rule_update.log 2>&1 &
POSTVERSION=2.3.21
}
post_2.3.2X_to_2.3.30() {
so-playbook-sigma-refresh >> /root/soup_playbook_sigma_refresh.log 2>&1 &
POSTVERSION=2.3.30
}
post_2.3.30_to_2.3.40() {
so-playbook-sigma-refresh >> /root/soup_playbook_sigma_refresh.log 2>&1 &
so-kibana-space-defaults
POSTVERSION=2.3.40
}
rc1_to_rc2() {
# Move the static file to global.sls
@@ -304,7 +306,6 @@ rc1_to_rc2() {
done </tmp/nodes.txt
INSTALLEDVERSION=rc.2
}
rc2_to_rc3() {
@@ -385,7 +386,6 @@ up_2.3.0_to_2.3.20(){
fi
INSTALLEDVERSION=2.3.20
}
up_2.3.2X_to_2.3.30() {
@@ -410,16 +410,26 @@ up_2.3.2X_to_2.3.30() {
check_log_size_limit
}
space_check() {
# Check to see if there is enough space
verify_upgradespace() {
CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//')
if [ "$CURRENTSPACE" -lt "10" ]; then
echo "You are low on disk space. Upgrade will try and clean up space.";
clean_dockers
echo "You are low on disk space."
return 1
else
echo "Plenty of space for upgrading"
return 0
fi
}
upgrade_space() {
if ! verify_upgradespace; then
clean_dockers
if ! verify_upgradespace; then
echo "There is not enough space to perform the upgrade. Please free up space and try again"
exit 1
fi
else
echo "You have enough space for upgrade. Proceeding with soup."
fi
}
thehive_maint() {
@@ -548,6 +558,7 @@ verify_latest_update_script() {
}
main () {
echo "### Preparing soup at `date` ###"
while getopts ":b" opt; do
case "$opt" in
b ) # process option b
@@ -558,7 +569,8 @@ while getopts ":b" opt; do
exit 1
fi
;;
\? ) echo "Usage: cmd [-b]"
\? )
echo "Usage: cmd [-b]"
;;
esac
done
@@ -573,6 +585,8 @@ check_airgap
echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
echo ""
set_os
set_palette
check_elastic_license
echo ""
if [ $is_airgap -eq 0 ]; then
# Let's mount the ISO since this is airgap
@@ -599,7 +613,7 @@ fi
echo "Let's see if we need to update Security Onion."
upgrade_check
space_check
upgrade_space
echo "Checking for Salt Master and Minion updates."
upgrade_check_salt
@@ -649,8 +663,7 @@ else
echo ""
fi
echo "Making pillar changes."
pillar_changes
preupgrade_changes
echo ""
if [ $is_airgap -eq 0 ]; then
@@ -704,7 +717,7 @@ echo "Starting Salt Master service."
systemctl start salt-master
echo "Running a highstate. This could take several minutes."
salt-call state.highstate -l info queue=True
playbook
postupgrade_changes
unmount_update
thehive_maint
@@ -741,6 +754,7 @@ NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l)
if [ $NUM_MINIONS -gt 1 ]; then
cat << EOF
This appears to be a distributed deployment. Other nodes should update themselves at the next Salt highstate (typically within 15 minutes). Do not manually restart anything until you know that all the search/heavy nodes in your deployment are updated. This is especially important if you are using true clustering for Elasticsearch.
Each minion is on a random 15 minute check-in period and things like network bandwidth can be a factor in how long the actual upgrade takes. If you have a heavy node on a slow link, it is going to take a while to get the containers to it. Depending on what changes happened between the versions, Elasticsearch might not be able to talk to said heavy node until the update is complete.
@@ -751,7 +765,23 @@ For more information, please see https://docs.securityonion.net/en/2.3/soup.html
EOF
fi
echo "### soup has been served at `date` ###"
}
main "$@" | tee /dev/fd/3
cat << EOF
SOUP - Security Onion UPdater
Please review the following for more information about the update process and recent updates:
https://docs.securityonion.net/soup
https://blog.securityonion.net
Please note that soup only updates Security Onion components and does NOT update the underlying operating system (OS). When you installed Security Onion, there was an option to automatically update the OS packages. If you did not enable this option, then you will want to ensure that the OS is fully updated before running soup.
Press Enter to continue or Ctrl-C to cancel.
EOF
read input
main "$@" | tee -a $SOUP_LOG

3
salt/curator/files/bin/so-curator-closed-delete-delete
View File

@@ -4,12 +4,11 @@
{%- if grains['role'] in ['so-node', 'so-heavynode'] %}
{%- set ELASTICSEARCH_HOST = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- set ELASTICSEARCH_PORT = salt['pillar.get']('elasticsearch:es_port', '') -%}
{%- set LOG_SIZE_LIMIT = salt['pillar.get']('elasticsearch:log_size_limit', '') -%}
{%- elif grains['role'] in ['so-eval', 'so-managersearch', 'so-standalone'] %}
{%- set ELASTICSEARCH_HOST = salt['pillar.get']('manager:mainip', '') -%}
{%- set ELASTICSEARCH_PORT = salt['pillar.get']('manager:es_port', '') -%}
{%- set LOG_SIZE_LIMIT = salt['pillar.get']('manager:log_size_limit', '') -%}
{%- endif -%}
{%- set LOG_SIZE_LIMIT = salt['pillar.get']('elasticsearch:log_size_limit', '') -%}
# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
#

83
salt/docker_clean/init.sls
View File

@@ -1,86 +1,9 @@
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls in allowed_states %}
{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
{% set MANAGER = salt['grains.get']('master') %}
{% set OLDVERSIONS = ['2.0.0-rc.1','2.0.1-rc.1','2.0.2-rc.1','2.0.3-rc.1','2.1.0-rc.2','2.2.0-rc.3','2.3.0','2.3.1','2.3.2','2.3.10','2.3.20']%}
{% for VERSION in OLDVERSIONS %}
remove_images_{{ VERSION }}:
docker_image.absent:
- force: True
- images:
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-acng:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-thehive-cortex:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-curator:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-domainstats:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elastalert:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elasticsearch:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-fleet:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-fleet-launcher:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-freqserver:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-grafana:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-idstools:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-influxdb:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-kibana:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-kratos:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-logstash:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-minio:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-mysql:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-nginx:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-playbook:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-redis:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-soc:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-soctopus:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-steno:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-strelka-frontend:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-strelka-manager:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-strelka-backend:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-strelka-filestream:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-suricata:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-telegraf:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-thehive:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-thehive-es:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-wazuh:{{ VERSION }}'
- '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-zeek:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-acng:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-thehive-cortex:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-curator:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-domainstats:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-elastalert:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-elasticsearch:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-filebeat:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-fleet:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-fleet-launcher:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-freqserver:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-grafana:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-idstools:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-influxdb:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-kibana:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-kratos:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-logstash:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-minio:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-mysql:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-nginx:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-pcaptools:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-playbook:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-redis:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-soc:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-soctopus:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-steno:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-strelka-frontend:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-strelka-manager:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-strelka-backend:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-strelka-filestream:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-suricata:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-telegraf:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-thehive:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-thehive-es:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-wazuh:{{ VERSION }}'
- '{{ MANAGER }}:5000/securityonion/so-zeek:{{ VERSION }}'
{% endfor %}
prune_images:
cmd.run:
- name: so-docker-prune
{% else %}

5
salt/elastalert/files/modules/so/playbook-es.py
View File

@@ -4,6 +4,9 @@ from time import gmtime, strftime
import requests,json
from elastalert.alerts import Alerter
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
class PlaybookESAlerter(Alerter):
"""
Use matched data to create alerts in elasticsearch
@@ -17,7 +20,7 @@ class PlaybookESAlerter(Alerter):
timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S", gmtime())
headers = {"Content-Type": "application/json"}
payload = {"rule": { "name": self.rule['play_title'],"case_template": self.rule['play_id'],"uuid": self.rule['play_id'],"category": self.rule['rule.category']},"event":{ "severity": self.rule['event.severity'],"module": self.rule['event.module'],"dataset": self.rule['event.dataset'],"severity_label": self.rule['sigma_level']},"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"event_data": match, "@timestamp": timestamp}
url = f"http://{self.rule['elasticsearch_host']}/so-playbook-alerts-{today}/_doc/"
url = f"https://{self.rule['elasticsearch_host']}/so-playbook-alerts-{today}/_doc/"
requests.post(url, data=json.dumps(payload), headers=headers, verify=False)
def get_info(self):

2
salt/elasticsearch/files/ingest/common
View File

@@ -32,8 +32,6 @@
{ "rename": { "field": "category", "target_field": "event.category", "ignore_failure": true, "ignore_missing": true } },
{ "rename": { "field": "message2.community_id", "target_field": "network.community_id", "ignore_failure": true, "ignore_missing": true } },
{ "lowercase": { "field": "event.dataset", "ignore_failure": true, "ignore_missing": true } },
{ "convert": { "field": "destination.port", "type": "integer", "ignore_failure": true, "ignore_missing": true } },
{ "convert": { "field": "source.port", "type": "integer", "ignore_failure": true, "ignore_missing": true } },
{ "convert": { "field": "log.id.uid", "type": "string", "ignore_failure": true, "ignore_missing": true } },
{ "convert": { "field": "agent.id", "type": "string", "ignore_failure": true, "ignore_missing": true } },
{ "convert": { "field": "event.severity", "type": "integer", "ignore_failure": true, "ignore_missing": true } },

19
salt/elasticsearch/templates/so/so-common-template.json
View File

@@ -51,16 +51,29 @@
"match_mapping_type": "string",
"path_match": "*.ip",
"mapping": {
"type": "ip"
"type": "ip",
"fields" : {
"keyword" : {
"ignore_above" : 45,
"type" : "keyword"
}
}
}
}
},
{
"port": {
"match_mapping_type": "string",
"path_match": "*.port",
"mapping": {
"type": "integer"
"type": "integer",
"fields" : {
"keyword" : {
"ignore_above" : 6,
"type" : "keyword"
}
}
}
}
},

41
salt/firewall/init.sls
View File

@@ -26,15 +26,6 @@ iptables_fix_fwd:
- position: 1
- target: DOCKER-USER
# Allow related/established sessions
iptables_allow_established:
iptables.append:
- table: filter
- chain: INPUT
- jump: ACCEPT
- match: conntrack
- ctstate: 'RELATED,ESTABLISHED'
# I like pings
iptables_allow_pings:
iptables.append:
@@ -77,17 +68,6 @@ enable_docker_user_fw_policy:
- out-interface: docker0
- position: 1
enable_docker_user_established:
iptables.insert:
- table: filter
- chain: DOCKER-USER
- jump: ACCEPT
- in-interface: '!docker0'
- out-interface: docker0
- position: 1
- match: conntrack
- ctstate: 'RELATED,ESTABLISHED'
{% set count = namespace(value=0) %}
{% for chain, hg in assigned_hostgroups.chain.items() %}
{% for hostgroup, portgroups in assigned_hostgroups.chain[chain].hostgroups.items() %}
@@ -120,6 +100,27 @@ enable_docker_user_established:
{% endfor %}
{% endfor %}
# Allow related/established sessions
iptables_allow_established:
iptables.insert:
- table: filter
- chain: INPUT
- jump: ACCEPT
- position: 1
- match: conntrack
- ctstate: 'RELATED,ESTABLISHED'
enable_docker_user_established:
iptables.insert:
- table: filter
- chain: DOCKER-USER
- jump: ACCEPT
- in-interface: '!docker0'
- out-interface: docker0
- position: 1
- match: conntrack
- ctstate: 'RELATED,ESTABLISHED'
# Block icmp timestamp response
block_icmp_timestamp_reply:
iptables.append:

42
salt/idstools/init.sls
View File

@@ -19,13 +19,12 @@
{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
{% set MANAGER = salt['grains.get']('master') %}
{% set ENGINE = salt['pillar.get']('global:mdengine', '') %}
{% set proxy = salt['pillar.get']('manager:proxy') %}
include:
- idstools.sync_files
# IDSTools Setup
idstoolsdir:
file.directory:
- name: /opt/so/conf/idstools/etc
- user: 939
- group: 939
- makedirs: True
idstoolslogdir:
file.directory:
@@ -34,14 +33,6 @@ idstoolslogdir:
- group: 939
- makedirs: True
idstoolsetcsync:
file.recurse:
- name: /opt/so/conf/idstools/etc
- source: salt://idstools/etc
- user: 939
- group: 939
- template: jinja
so-ruleupdatecron:
cron.present:
- name: /usr/sbin/so-rule-update > /opt/so/log/idstools/download.log 2>&1
@@ -49,28 +40,17 @@ so-ruleupdatecron:
- minute: '1'
- hour: '7'
rulesdir:
file.directory:
- name: /opt/so/rules/nids
- user: 939
- group: 939
- makedirs: True
# Don't show changes because all.rules can be large
synclocalnidsrules:
file.recurse:
- name: /opt/so/rules/nids/
- source: salt://idstools/
- user: 939
- group: 939
- show_changes: False
- include_pat: 'E@.rules'
so-idstools:
docker_container.running:
- image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-idstools:{{ VERSION }}
- hostname: so-idstools
- user: socore
{% if proxy %}
- environment:
- http_proxy={{ proxy }}
- https_proxy={{ proxy }}
- no_proxy={{ salt['pillar.get']('manager:no_proxy') }}
{% endif %}
- binds:
- /opt/so/conf/idstools/etc:/opt/so/idstools/etc:ro
- /opt/so/rules/nids:/opt/so/rules/nids:rw

46
salt/idstools/sync_files.sls Normal file
View File

@@ -0,0 +1,46 @@
# Copyright 2014,2015,2016,2017,2018 Security Onion Solutions, LLC
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
idstoolsdir:
file.directory:
- name: /opt/so/conf/idstools/etc
- user: 939
- group: 939
- makedirs: True
idstoolsetcsync:
file.recurse:
- name: /opt/so/conf/idstools/etc
- source: salt://idstools/etc
- user: 939
- group: 939
- template: jinja
rulesdir:
file.directory:
- name: /opt/so/rules/nids
- user: 939
- group: 939
- makedirs: True
# Don't show changes because all.rules can be large
synclocalnidsrules:
file.recurse:
- name: /opt/so/rules/nids/
- source: salt://idstools/
- user: 939
- group: 939
- show_changes: False
- include_pat: 'E@.rules'

2
salt/kibana/bin/so-kibana-config-load
View File

@@ -23,4 +23,4 @@ wait_for_web_response "http://localhost:5601/app/kibana" "Elastic"
THECOOKIE=$(curl -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
# Load saved objects
curl -b "sid=$THECOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson > /dev/null 2>&1
curl -b "sid=$THECOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson >> /opt/so/log/kibana/misc.log

1464
salt/kibana/files/saved_objects.ndjson
View File

File diff suppressed because one or more lines are too long

2
salt/nodered/files/so_flows.json
View File

File diff suppressed because one or more lines are too long

2
salt/reactor/fleet.sls
View File

@@ -89,7 +89,7 @@ def run():
# Update the Fleet host in the static pillar
for line in fileinput.input(STATICFILE, inplace=True):
line = re.sub(r'fleet_custom_hostname:.*\n', f"fleet_custom_hostname: {CUSTOMHOSTNAME}", line.rstrip())
line = re.sub(r'fleet_custom_hostname:.*$', f"fleet_custom_hostname: {CUSTOMHOSTNAME}", line.rstrip())
print(line)
return {}

93
salt/soc/files/soc/changes.json
View File

@@ -1,52 +1,49 @@
{
"title": "Security Onion 2.3.30 is here!",
"title": "Security Onion 2.3.40 is here!",
"changes": [
{ "summary": "Zeek is now at version 3.0.13." },
{ "summary": "CyberChef is now at version 9.27.2." },
{ "summary": "Elastic components are now at version 7.10.2. This is the last version that uses the Apache license." },
{ "summary": "Suricata is now at version 6.0.1." },
{ "summary": "Salt is now at version 3002.5." },
{ "summary": "Suricata metadata parsing is now vastly improved." },
{ "summary": "If you choose Suricata for metadata parsing, it will now extract files from the network and send them to Strelka. You can add additional mime types <a href='https://github.com/Security-Onion-Solutions/securityonion/blob/dev/salt/idstools/sorules/extraction.rules'>here</a>." },
{ "summary": "It is now possible to filter Suricata events from being written to the logs. This is a new Suricata 6 feature. We have included some examples <a href='https://github.com/Security-Onion-Solutions/securityonion/blob/dev/salt/idstools/sorules/filters.rules'>here</a>." },
{ "summary": "The Kratos docker container will now perform DNS lookups locally before reaching out to the network DNS provider." },
{ "summary": "Network configuration is now more compatible with manually configured OpenVPN or Wireguard VPN interfaces." },
{ "summary": "<code>so-sensor-clean</code> will no longer spawn multiple instances." },
{ "summary": "Suricata eve.json logs will now be cleaned up after 7 days. This can be changed via the pillar setting." },
{ "summary": "Fixed a security issue where the backup directory had improper file permissions." },
{ "summary": "The automated backup script on the manager now backs up all keys along with the salt configurations. Backup retention is now set to 7 days." },
{ "summary": "Strelka logs are now being rotated properly." },
{ "summary": "Elastalert can now be customized via a pillar." },
{ "summary": "Introduced new script <code>so-monitor-add</code> that allows the user to easily add interfaces to the bond for monitoring." },
{ "summary": "Setup now validates all user input fields to give up-front feedback if an entered value is invalid." },
{ "summary": "There have been several changes to improve install reliability. Many install steps have had their validation processes reworked to ensure that required tasks have been completed before moving on to the next step of the install." },
{ "summary": "Users are now warned if they try to set <i>securityonion</i> as their hostname." },
{ "summary": "The ISO should now identify xvda and nvme devices as install targets." },
{ "summary": "At the end of the first stage of the ISO setup, the ISO device should properly unmount and eject." },
{ "summary": "The text selection of choosing Suricata vs Zeek for metadata is now more descriptive." },
{ "summary": "The logic for properly setting the <code>LOG_SIZE_LIMIT</code> variable has been improved." },
{ "summary": "When installing on Ubuntu, Setup will now wait for cloud init to complete before trying to start the install of packages." },
{ "summary": "The firewall state runs considerably faster now." },
{ "summary": "ICMP timestamps are now disabled." },
{ "summary": "Copyright dates on all Security Onion specific files have been updated." },
{ "summary": "<code>so-tcpreplay</code> (and indirectly <code>so-test</code>) should now work properly." },
{ "summary": "The Zeek packet loss script is now more accurate." },
{ "summary": "Grafana now includes an estimated EPS graph for events ingested on the manager." },
{ "summary": "Updated Elastalert to release 0.2.4-alt2 based on the <a href='https://github.com/jertel/elastalert'>jertel/elastalert</a> alt branch." },
{ "summary": "Pivots from Alerts/Hunts to action links will properly URI encode values." },
{ "summary": "Hunt timeline graph will properly scale the data point interval based on the search date range." },
{ "summary": "Grid interface will properly show <i>Search</i> as the node type instead of <i>so-node</i>." },
{ "summary": "Import node now supports airgap environments." },
{ "summary": "The so-mysql container will now show <i>healthy</i> when viewing the docker ps output." },
{ "summary": "The Soctopus configuration now uses private IPs instead of public IPs, allowing network communications to succeed within the grid." },
{ "summary": "The Correlate action in Hunt now groups the OR filters together to ensure subsequent user-added filters are correctly ANDed to the entire OR group." },
{ "summary": "Add support to <code>so-firewall</code> script to display existing port groups and host groups." },
{ "summary": "TheHive initialization during Security Onion setup will now properly check for a running ES instance and will retry connectivity checks to TheHive before proceeding." },
{ "summary": "Changes to the <i>.security</i> analyzer yields more accurate query results when using Playbook." },
{ "summary": "Several Hunt queries have been updated." },
{ "summary": "The pfSense firewall log parser has been updated to improve compatibility." },
{ "summary": "Kibana dashboard hyperlinks have been updated for faster navigation." },
{ "summary": "Added a new <code>so-rule</code> script to make it easier to disable, enable, and modify SIDs." },
{ "summary": "ISO now gives the option to just configure the network during setup." }
{ "summary": "FEATURE: Add option for HTTP Method Specification/POST to Hunt/Alerts Actions <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/2904\">#2904</a>" },
{ "summary": "FEATURE: Add option to configure proxy for various tools used during setup + persist the proxy configuration <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/529\">#529</a>" },
{ "summary": "FEATURE: Alerts/Hunt - Provide method for base64-encoding pivot value <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/1749\">#1749</a>" },
{ "summary": "FEATURE: Allow users to customize links in SOC <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/1248\">#1248</a>" },
{ "summary": "FEATURE: Display user who requested PCAP in SOC <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/2775\">#2775</a>" },
{ "summary": "FEATURE: Make SOC browser app connection timeouts adjustable <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/2408\">#2408</a>" },
{ "summary": "FEATURE: Move to FleetDM <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3483\">#3483</a>" },
{ "summary": "FEATURE: Reduce field cache expiration from 1d to 5m, and expose value as a salt pillar <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3537\">#3537</a>" },
{ "summary": "FEATURE: Refactor docker_clean salt state to use loop w/ inspection instead of hardcoded image list <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3113\">#3113</a>" },
{ "summary": "FEATURE: Run so-ssh-harden during setup <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/1932\">#1932</a>" },
{ "summary": "FEATURE: SOC should only display links to tools that are enabled <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/1643\">#1643</a>" },
{ "summary": "FEATURE: Update Sigmac Osquery Field Mappings <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3137\">#3137</a>" },
{ "summary": "FEATURE: User must accept the Elastic licence during setup <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3233\">#3233</a>" },
{ "summary": "FEATURE: soup should output more guidance for distributed deployments at the end <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3340\">#3340</a>" },
{ "summary": "FEATURE: soup should provide some initial information and then prompt the user to continue <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3486\">#3486</a>" },
{ "summary": "FIX: Add cronjob for so-suricata-eve-clean script <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3515\">#3515</a>" },
{ "summary": "FIX: Change Elasticsearch heap formula <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/1686\">#1686</a>" },
{ "summary": "FIX: Create a post install version loop in soup <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3102\">#3102</a>" },
{ "summary": "FIX: Custom Kibana settings are not being applied properly on upgrades <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3254\">#3254</a>" },
{ "summary": "FIX: Hunt query issues with quotes <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3320\">#3320</a>" },
{ "summary": "FIX: IP Addresses don't work with .security <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3327\">#3327</a>" },
{ "summary": "FIX: Improve DHCP leases query in Hunt <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3395\">#3395</a>" },
{ "summary": "FIX: Improve Setup verbiage <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3422\">#3422</a>" },
{ "summary": "FIX: Improve Suricata DHCP logging and parsing <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3397\">#3397</a>" },
{ "summary": "FIX: Keep RELATED,ESTABLISHED rules at the top of iptables chains <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3288\">#3288</a>" },
{ "summary": "FIX: Populate http.status_message field <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3408\">#3408</a>" },
{ "summary": "FIX: Remove 'types removal' deprecation messages from elastic log. <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3345\">#3345</a>" },
{ "summary": "FIX: Reword + fix formatting on ES data storage prompt <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3205\">#3205</a>" },
{ "summary": "FIX: SMTP shoud read SNMP on Kibana SNMP view <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3413\">#3413</a>" },
{ "summary": "FIX: Sensors can temporarily show offline while processing large PCAP jobs <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3279\">#3279</a>" },
{ "summary": "FIX: Soup should log to the screen as well as to a file <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3467\">#3467</a>" },
{ "summary": "FIX: Strelka port 57314 not immediately relinquished upon restart <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3457\">#3457</a>" },
{ "summary": "FIX: Switch SOC to pull from fieldcaps API due to field caching changes in Kibana 7.11 <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3502\">#3502</a>" },
{ "summary": "FIX: Syntax error in /etc/sysctl.d/99-reserved-ports.conf <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3308\">#3308</a>" },
{ "summary": "FIX: Telegraf hardcoded to use https and is not aware of elasticsearch features <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/2061\">#2061</a>" },
{ "summary": "FIX: Zeek Index Close and Delete Count for curator <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3274\">#3274</a>" },
{ "summary": "FIX: so-cortex-user-add and so-cortex-user-enable use wrong pillar value for api key <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3388\">#3388</a>" },
{ "summary": "FIX: so-rule does not completely apply change <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3289\">#3289</a>" },
{ "summary": "FIX: soup should recheck disk space after it tries to clean up. <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3235\">#3235</a>" },
{ "summary": "UPGRADE: Elastic 7.11.2 <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3389\">#3389</a>" },
{ "summary": "UPGRADE: Suricata 6.0.2 <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3217\">#3217</a>" },
{ "summary": "UPGRADE: Zeek 4 <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3216\">#3216</a>" },
{ "summary": "UPGRADE: Zeek container to use Python 3 <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/1113\">#1113</a>" },
{ "summary": "UPGRADE: docker-ce to latest <a href=\"https://github.com/Security-Onion-Solutions/securityonion/issues/3493\">#3493</a>" }
]
}

23
salt/soc/files/soc/soc.json
View File

@@ -1,17 +1,23 @@
{%- set MANAGERIP = salt['pillar.get']('global:managerip', '') %}
{%- set SENSORONIKEY = salt['pillar.get']('global:sensoronikey', '') %}
{%- set THEHIVEKEY = salt['pillar.get']('global:hivekey', '') %}
{%- set PLAYBOOK = salt['pillar.get']('manager:playbook', '0') %}
{%- set THEHIVE = salt['pillar.get']('manager:thehive', '0') %}
{%- set OSQUERY = salt['pillar.get']('manager:osquery', '0') %}
{%- set GRAFANA = salt['pillar.get']('manager:grafana', '0') %}
{%- set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') %}
{%- set API_TIMEOUT = salt['pillar.get']('sensoroni:api_timeout_ms', 0) %}
{%- set WEBSOCKET_TIMEOUT = salt['pillar.get']('sensoroni:websocket_timeout_ms', 0) %}
{%- set TIP_TIMEOUT = salt['pillar.get']('sensoroni:tip_timeout_ms', 0) %}
{%- set CACHE_EXPIRATION = salt['pillar.get']('sensoroni:cache_expiration_ms', 0) %}
{%- set ES_FIELDCAPS_CACHE = salt['pillar.get']('sensoroni:es_fieldcaps_cache_ms', '300000') %}
{%- import_json "soc/files/soc/alerts.queries.json" as alerts_queries %}
{%- import_json "soc/files/soc/alerts.actions.json" as alerts_actions %}
{%- import_json "soc/files/soc/alerts.eventfields.json" as alerts_eventfields %}
{%- import_json "soc/files/soc/hunt.queries.json" as hunt_queries %}
{%- import_json "soc/files/soc/hunt.actions.json" as hunt_actions %}
{%- import_json "soc/files/soc/hunt.eventfields.json" as hunt_eventfields %}
{%- import_json "soc/files/soc/tools.json" as tools %}
{%- set DNET = salt['pillar.get']('global:dockernet', '172.17.0.0') %}
{
@@ -44,6 +50,7 @@
{%- endif %}
"username": "",
"password": "",
"cacheMs": {{ ES_FIELDCAPS_CACHE }},
"verifyCert": false
},
"sostatus": {
@@ -74,6 +81,22 @@
"webSocketTimeoutMs": {{ WEBSOCKET_TIMEOUT }},
"tipTimeoutMs": {{ TIP_TIMEOUT }},
"cacheExpirationMs": {{ CACHE_EXPIRATION }},
"inactiveTools": [
{%- if PLAYBOOK == 0 %}
"toolPlaybook",
{%- endif %}
{%- if THEHIVE == 0 %}
"toolTheHive",
{%- endif %}
{%- if OSQUERY == 0 %}
"toolFleet",
{%- endif %}
{%- if GRAFANA == 0 %}
"toolGrafana",
{%- endif %}
"toolUnused"
],
"tools": {{ tools | json }},
"hunt": {
"advanced": true,
"groupItemsPerPage": 10,

9
salt/soc/files/soc/tools.json Normal file
View File

@@ -0,0 +1,9 @@
[
{ "name": "toolKibana", "description": "toolKibanaHelp", "icon": "fa-external-link-alt", "target": "so-kibana", "link": "/kibana/" },
{ "name": "toolGrafana", "description": "toolGrafanaHelp", "icon": "fa-external-link-alt", "target": "so-grafana", "link": "/grafana/d/so_overview" },
{ "name": "toolCyberchef", "description": "toolCyberchefHelp", "icon": "fa-external-link-alt", "target": "so-cyberchef", "link": "/cyberchef/" },
{ "name": "toolPlaybook", "description": "toolPlaybookHelp", "icon": "fa-external-link-alt", "target": "so-playbook", "link": "/playbook/projects/detection-playbooks/issues/" },
{ "name": "toolFleet", "description": "toolFleetHelp", "icon": "fa-external-link-alt", "target": "so-fleet", "link": "/fleet/" },
{ "name": "toolTheHive", "description": "toolTheHiveHelp", "icon": "fa-external-link-alt", "target": "so-thehive", "link": "/thehive/" },
{ "name": "toolNavigator", "description": "toolNavigatorHelp", "icon": "fa-external-link-alt", "target": "so-navigator", "link": "/navigator/" }
]

3
salt/strelka/files/filestream/filestream.yaml
View File

@@ -19,7 +19,8 @@ files:
- '/nsm/strelka/unprocessed/*'
delete: false
gatekeeper: true
processed: '/nsm/strelka/processed'
response:
report: 5s
delta: 5s
staging: '/nsm/strelka/processed'
staging: '/nsm/strelka/staging'

9
salt/strelka/init.sls
View File

@@ -86,6 +86,13 @@ strelkaprocessed:
- group: 939
- makedirs: True
strelkastaging:
file.directory:
- name: /nsm/strelka/staging
- user: 939
- group: 939
- makedirs: True
strelkaunprocessed:
file.directory:
- name: /nsm/strelka/unprocessed
@@ -96,7 +103,7 @@ strelkaunprocessed:
# Check to see if Strelka frontend port is available
strelkaportavailable:
cmd.run:
- name: netstat -utanp | grep ":57314" | grep -qv docker && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314. Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0
- name: netstat -utanp | grep ":57314" | grep -qvE 'docker|TIME_WAIT' && PROCESS=$(netstat -utanp | grep ":57314" | uniq) && echo "Another process ($PROCESS) appears to be using port 57314. Please terminate this process, or reboot to ensure a clean state so that Strelka can start properly." && exit 1 || exit 0
strelka_coordinator:
docker_container.running:

20
salt/suricata/init.sls
View File

@@ -179,6 +179,26 @@ disable_so-suricata_so-status.conf:
- month: '*'
- dayweek: '*'
so-suricata-eve-clean:
file.managed:
- name: /usr/sbin/so-suricata-eve-clean
- user: root
- group: root
- mode: 755
- template: jinja
- source: salt://suricata/cron/so-suricata-eve-clean
# Add eve clean cron
clean_suricata_eve_files:
cron.present:
- name: /usr/sbin/so-suricata-eve-clean > /dev/null 2>&1
- user: root
- minute: '*/5'
- hour: '*'
- daymonth: '*'
- month: '*'
- dayweek: '*'
{% else %}
{{sls}}_state_not_allowed:

5
salt/telegraf/etc/telegraf.conf
View File

@@ -618,11 +618,8 @@
# # Read stats from one or more Elasticsearch servers or clusters
{% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone'] %}
[[inputs.elasticsearch]]
# ## specify a list of one or more Elasticsearch servers
# # you can add username and password to your url to use basic authentication:
# # servers = ["http://user:pass@localhost:9200"]
servers = ["https://{{ MANAGER }}:9200"]
insecure_skip_verify = true
{% elif grains['role'] in ['so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode'] %}
[[inputs.elasticsearch]]
servers = ["https://{{ NODEIP }}:9200"]

6
salt/yum/etc/yum.conf.jinja
View File

@@ -1,3 +1,4 @@
{% set proxy = salt['pillar.get']('manager:proxy') -%}
[main]
cachedir=/var/cache/yum/$basearch/$releasever
keepcache=0
@@ -11,7 +12,8 @@ installonly_limit={{ salt['pillar.get']('yum:config:installonly_limit', 2) }}
bugtracker_url=http://bugs.centos.org/set_project.php?project_id=23&ref=http://bugs.centos.org/bug_report_page.php?category=yum
distroverpkg=centos-release
clean_requirements_on_remove=1
{% if (grains['role'] not in ['so-eval','so-managersearch', 'so-manager', 'so-standalone']) and salt['pillar.get']('global:managerupdate', '0') %}
{% if (grains['role'] not in ['so-eval','so-managersearch', 'so-manager', 'so-standalone']) and salt['pillar.get']('global:managerupdate', '0') -%}
proxy=http://{{ salt['pillar.get']('yum:config:proxy', salt['config.get']('master')) }}:3142
{% elif proxy -%}
proxy={{ proxy }}
{% endif %}

78
setup/automation/standalone-net-centos-proxy Normal file
View File

@@ -0,0 +1,78 @@
#!/bin/bash
# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
TESTING=true
# address_type=DHCP
ADMINUSER=onionuser
ADMINPASS1=onionuser
ADMINPASS2=onionuser
ALLOW_CIDR=0.0.0.0/0
ALLOW_ROLE=a
BASICZEEK=2
BASICSURI=2
# BLOGS=
BNICS=eth1
ZEEKVERSION=ZEEK
# CURCLOSEDAYS=
# EVALADVANCED=BASIC
GRAFANA=1
# HELIXAPIKEY=
HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
HNSENSOR=inherit
HOSTNAME=standalone
install_type=STANDALONE
# LSINPUTBATCHCOUNT=
# LSINPUTTHREADS=
# LSPIPELINEBATCH=
# LSPIPELINEWORKERS=
MANAGERADV=BASIC
MANAGERUPDATES=1
# MDNS=
# MGATEWAY=
# MIP=
# MMASK=
MNIC=eth0
# MSEARCH=
# MSRV=
# MTU=
NIDS=Suricata
# NODE_ES_HEAP_SIZE=
# NODE_LS_HEAP_SIZE=
NODESETUP=NODEBASIC
NSMSETUP=BASIC
NODEUPDATES=MANAGER
# OINKCODE=
OSQUERY=1
# PATCHSCHEDULEDAYS=
# PATCHSCHEDULEHOURS=
PATCHSCHEDULENAME=auto
PLAYBOOK=1
so_proxy=http://onionuser:0n10nus3r@10.66.166.30:3128
# REDIRECTHOST=
REDIRECTINFO=IP
RULESETUP=ETOPEN
# SHARDCOUNT=
# SKIP_REBOOT=
SOREMOTEPASS1=onionuser
SOREMOTEPASS2=onionuser
STRELKA=1
THEHIVE=1
WAZUH=1
WEBUSER=onionuser@somewhere.invalid
WEBPASSWD1=0n10nus3r
WEBPASSWD2=0n10nus3r

2
setup/proxies/docker.conf
View File

@@ -1,2 +0,0 @@
[Service]
ExecStart=/usr/bin/dockerd /usr/bin/dockerd -H fd:// --registry-mirror "$proxy_addr"

240
setup/so-functions
View File

@@ -535,6 +535,56 @@ collect_patch_schedule_name_import() {
done
}
collect_proxy() {
[[ -n $TESTING ]] && return
collect_proxy_details
while ! proxy_validate; do
if whiptail_invalid_proxy; then
collect_proxy_details no_ask
else
so_proxy=""
break
fi
done
}
collect_proxy_details() {
local ask=${1:-true}
local use_proxy
if [[ $ask != true ]]; then
use_proxy=0
else
whiptail_proxy_ask
use_proxy=$?
fi
if [[ $use_proxy == 0 ]]; then
whiptail_proxy_addr "$proxy_addr"
while ! valid_proxy "$proxy_addr"; do
whiptail_invalid_input
whiptail_proxy_addr "$proxy_addr"
done
if whiptail_proxy_auth_ask; then
whiptail_proxy_auth_user "$proxy_user"
whiptail_proxy_auth_pass "$proxy_pass"
local url_prefixes=( 'http://' 'https://' )
for prefix in "${url_prefixes[@]}"; do
if echo "$proxy_addr" | grep -q "$prefix"; then
local proxy=${proxy_addr#"$prefix"}
so_proxy="${prefix}${proxy_user}:${proxy_pass}@${proxy}"
break
fi
done
else
so_proxy="$proxy_addr"
fi
export proxy
fi
}
collect_redirect_host() {
whiptail_set_redirect_host "$HOSTNAME"
@@ -691,10 +741,10 @@ check_requirements() {
else
req_storage=100
fi
if (( $(echo "$free_space_root < $req_storage" | bc -l) )); then
if [[ $free_space_root -lt $req_storage ]]; then
whiptail_storage_requirements "/" "${free_space_root} GB" "${req_storage} GB"
fi
if (( $(echo "$free_space_nsm < $req_storage" | bc -l) )); then
if [[ $free_space_nsm -lt $req_storage ]]; then
whiptail_storage_requirements "/nsm" "${free_space_nsm} GB" "${req_storage} GB"
fi
else
@@ -703,7 +753,7 @@ check_requirements() {
else
req_storage=200
fi
if (( $(echo "$free_space_root < $req_storage" | bc -l) )); then
if [[ $free_space_root -lt $req_storage ]]; then
whiptail_storage_requirements "/" "${free_space_root} GB" "${req_storage} GB"
fi
fi
@@ -743,12 +793,14 @@ check_sos_appliance() {
compare_main_nic_ip() {
if ! [[ $MNIC =~ ^(tun|wg|vpn).*$ ]]; then
if [[ "$MAINIP" != "$MNIC_IP" ]]; then
error "[ERROR] Main gateway ($MAINIP) does not match ip address of managament NIC ($MNIC_IP)."
read -r -d '' message <<- EOM
The IP being routed by Linux is not the IP address assigned to the management interface ($MNIC).
This is not a supported configuration, please remediate and rerun setup.
EOM
whiptail --title "Security Onion Setup" --msgbox "$message" 10 75
[[ -n $TESTING ]] || whiptail --title "Security Onion Setup" --msgbox "$message" 10 75
kill -SIGINT "$(ps --pid $$ -oppid=)"; exit 1
fi
else
@@ -897,7 +949,7 @@ create_repo() {
}
detect_cloud() {
echo "Testing if setup is running on a cloud instance..." >> "$setup_log" 2>&1
echo "Testing if setup is running on a cloud instance..." | tee -a "$setup_log"
if ( curl --fail -s -m 5 http://169.254.169.254/latest/meta-data/instance-id > /dev/null ) || ( dmidecode -s bios-vendor | grep -q Google > /dev/null); then export is_cloud="true"; fi
}
@@ -939,36 +991,29 @@ detect_os() {
}
installer_prereq_packages() {
installer_progress_loop() {
local i=0
while true; do
[[ $i -lt 98 ]] && ((i++))
set_progress_str "$i" 'Checking that all required packages are installed and enabled...' nolog
[[ $i -gt 0 ]] && sleep 5s
done
}
installer_prereq_packages() {
if [ "$OS" == centos ]; then
# Print message to stdout so the user knows setup is doing something
echo "Installing required packages to run installer..."
# Install bind-utils so the host command exists
if [[ ! $is_iso ]]; then
if ! command -v host > /dev/null 2>&1; then
yum -y install bind-utils >> "$setup_log" 2>&1
fi
if ! command -v nmcli > /dev/null 2>&1; then
{
yum -y install NetworkManager;
systemctl enable NetworkManager;
systemctl start NetworkManager;
} >> "$setup_log" 2<&1
fi
if ! command -v bc > /dev/null 2>&1; then
yum -y install bc >> "$setup_log" 2>&1
fi
if ! yum versionlock > /dev/null 2>&1; then
yum -y install yum-plugin-versionlock >> "$setup_log" 2>&1
fi
else
if ! command -v nmcli > /dev/null 2>&1; then
yum -y install NetworkManager >> "$setup_log" 2>&1
fi
fi
logCmd "systemctl enable NetworkManager"
logCmd "systemctl start NetworkManager"
fi
elif [ "$OS" == ubuntu ]; then
# Print message to stdout so the user knows setup is doing something
echo "Installing required packages to run installer..."
retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1
# Install network manager so we can do interface stuff
if ! command -v nmcli > /dev/null 2>&1; then
@@ -978,7 +1023,7 @@ installer_prereq_packages() {
systemctl start NetworkManager
} >> "$setup_log" 2<&1
fi
retry 50 10 "apt-get -y install bc curl" >> "$setup_log" 2>&1 || exit 1
retry 50 10 "apt-get -y install curl" >> "$setup_log" 2>&1 || exit 1
fi
}
@@ -1044,10 +1089,11 @@ docker_install() {
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo;
fi
if [[ ! $is_iso ]]; then
yum -y install docker-ce-19.03.14-3.el7 containerd.io-1.2.13-3.2.el7;
yum -y install docker-ce-20.10.5-3.el7 containerd.io-1.4.4-3.1.el7;
fi
yum versionlock docker-ce-19.03.14-3.el7;
yum versionlock containerd.io-1.2.13-3.2.el7
yum versionlock docker-ce-20.10.5-3.el7;
yum versionlock docker-ce-cli-20.10.5-3.el7;
yum versionlock containerd.io-1.4.4-3.1.el7
} >> "$setup_log" 2>&1
else
@@ -1201,8 +1247,13 @@ es_heapsize() {
# https://www.elastic.co/guide/en/elasticsearch/guide/current/heap-sizing.html
ES_HEAP_SIZE="25000m"
else
# Set heap size to 25% of available memory
ES_HEAP_SIZE=$(( total_mem / 4 ))"m"
# Set heap size to 33% of available memory
ES_HEAP_SIZE=$(( total_mem / 3 ))
if [ "$ES_HEAP_SIZE" -ge 25001 ] ; then
ES_HEAP_SIZE="25000m"
else
ES_HEAP_SIZE=$ES_HEAP_SIZE"m"
fi
fi
export ES_HEAP_SIZE
@@ -1385,6 +1436,8 @@ install_cleanup() {
info "Removing so-setup permission entry from sudoers file"
sed -i '/so-setup/d' /etc/sudoers
fi
so-ssh-harden -q
}
import_registry_docker() {
@@ -1432,6 +1485,8 @@ manager_pillar() {
"manager:"\
" mainip: '$MAINIP'"\
" mainint: '$MNIC'"\
" proxy: '$so_proxy'"\
" no_proxy: '$no_proxy_string'"\
" esheap: '$ES_HEAP_SIZE'"\
" esclustername: '{{ grains.host }}'"\
" freq: 0"\
@@ -1446,7 +1501,6 @@ manager_pillar() {
printf '%s\n'\
" elastalert: 1"\
" es_port: $node_es_port"\
" log_size_limit: $log_size_limit"\
" cur_close_days: $CURCLOSEDAYS"\
" grafana: $GRAFANA"\
" osquery: $OSQUERY"\
@@ -1512,7 +1566,6 @@ manager_global() {
" hnmanager: '$HNMANAGER'"\
" ntpserver: '$NTPSERVER'"\
" dockernet: '$DOCKERNET'"\
" proxy: '$PROXY'"\
" mdengine: '$ZEEKVERSION'"\
" ids: '$NIDS'"\
" url_base: '$REDIRECTIT'"\
@@ -1642,8 +1695,8 @@ manager_global() {
" so-zeek:"\
" shards: 5"\
" warm: 7"\
" close: 365"\
" delete: 45"\
" close: 45"\
" delete: 365"\
"minio:"\
" access_key: '$ACCESS_KEY'"\
" access_secret: '$ACCESS_SECRET'"\
@@ -1695,7 +1748,6 @@ network_init() {
network_init_whiptail() {
case "$setup_type" in
'iso')
collect_hostname
whiptail_management_nic
whiptail_dhcp_or_static
@@ -1709,7 +1761,6 @@ network_init_whiptail() {
'network')
whiptail_network_notice
whiptail_dhcp_warn
collect_hostname
whiptail_management_nic
;;
esac
@@ -1777,6 +1828,22 @@ print_salt_state_apply() {
echo "Applying $state Salt state"
}
proxy_validate() {
local test_url="https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS"
proxy_test_err=$(curl -sS "$test_url" --proxy "$so_proxy" 2>&1)
local ret=$?
if [[ $ret != 0 ]]; then
error "Could not reach $test_url using proxy $so_proxy"
error "Received error: $proxy_test_err"
if [[ -n $TESTING ]]; then
error "Exiting setup"
kill -SIGINT "$(ps --pid $$ -oppid=)"; exit 1
fi
fi
return $ret
}
reserve_group_ids() {
# This is a hack to fix CentOS from taking group IDs that we need
groupadd -g 928 kratos
@@ -1870,6 +1937,24 @@ reinstall_init() {
} >> "$setup_log" 2>&1
}
reset_proxy() {
[[ -f /etc/profile.d/so-proxy.sh ]] && rm -f /etc/profile.d/so-proxy.sh
[[ -f /etc/systemd/system/docker.service.d/http-proxy.conf ]] && rm -f /etc/systemd/system/docker.service.d/http-proxy.conf
systemctl daemon-reload
command -v docker &> /dev/null && echo "Restarting Docker..." | tee -a "$setup_log" && systemctl restart docker
[[ -f /root/.docker/config.json ]] && rm -f /root/.docker/config.json
[[ -f /etc/gitconfig ]] && rm -f /etc/gitconfig
if [[ $OS == 'centos' ]]; then
sed -i "/proxy=/d" /etc/yum.conf
else
[[ -f /etc/apt/apt.conf.d/00-proxy.conf ]] && rm -f /etc/apt/apt.conf.d/00-proxy.conf
fi
}
backup_dir() {
dir=$1
backup_suffix=$2
@@ -1963,6 +2048,7 @@ saltify() {
python36-dateutil\
python36-m2crypto\
python36-mysql\
python36-packaging\
yum-utils\
device-mapper-persistent-data\
lvm2\
@@ -2051,9 +2137,9 @@ saltify() {
retry 50 10 "apt-get -y install salt-minion=3002.5+ds-1 salt-common=3002.5+ds-1" >> "$setup_log" 2>&1 || exit 1
retry 50 10 "apt-mark hold salt-minion salt-common" >> "$setup_log" 2>&1 || exit 1
if [[ $OSVER != 'xenial' ]]; then
retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb" >> "$setup_log" 2>&1 || exit 1
retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-packaging" >> "$setup_log" 2>&1 || exit 1
else
retry 50 10 "apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb" >> "$setup_log" 2>&1 || exit 1
retry 50 10 "apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb python-packaging" >> "$setup_log" 2>&1 || exit 1
fi
fi
}
@@ -2195,7 +2281,70 @@ set_main_ip() {
# Add /usr/sbin to everyone's path
set_path() {
echo "complete -cf sudo" > /etc/profile.d/securityonion.sh
echo "complete -cf sudo" >> /etc/profile.d/securityonion.sh
}
set_proxy() {
# Don't proxy localhost, local ip, and management ip
no_proxy_string="localhost, 127.0.0.1, ${MAINIP}, ${HOSTNAME}"
if [[ -n $MSRV ]] && [[ -n $MSRVIP ]];then
no_proxy_string="${no_proxy_string}, ${MSRVIP}, ${MSRV}"
fi
# Set proxy environment variables used by curl, wget, docker, and others
{
echo "export use_proxy=on"
echo "export http_proxy=\"${so_proxy}\""
echo "export https_proxy=\"\$http_proxy\""
echo "export ftp_proxy=\"\$http_proxy\""
echo "export no_proxy=\"${no_proxy_string}\""
} > /etc/profile.d/so-proxy.sh
source /etc/profile.d/so-proxy.sh
[[ -d '/etc/systemd/system/docker.service.d' ]] || mkdir -p /etc/systemd/system/docker.service.d
# Create proxy config for dockerd
printf '%s\n'\
"[Service]"\
"Environment=\"HTTP_PROXY=${so_proxy}\""\
"Environment=\"HTTPS_PROXY=${so_proxy}\""\
"Environment=\"NO_PROXY=${no_proxy_string}\"" > /etc/systemd/system/docker.service.d/http-proxy.conf
systemctl daemon-reload
command -v docker &> /dev/null && systemctl restart docker
# Create config.json for docker containers
[[ -d /root/.docker ]] || mkdir /root/.docker
printf '%s\n'\
"{"\
" \"proxies\":"\
" {"\
" \"default\":"\
" {"\
" \"httpProxy\":\"${so_proxy}\","\
" \"httpsProxy\":\"${so_proxy}\","\
" \"ftpProxy\":\"${so_proxy}\","\
" \"noProxy\":\"${no_proxy_string}\""\
" }"\
" }"\
"}" > /root/.docker/config.json
# Set proxy for package manager
if [ "$OS" = 'centos' ]; then
echo "proxy=$so_proxy" >> /etc/yum.conf
else
# Set it up so the updates roll through the manager
printf '%s\n'\
"Acquire::http::Proxy \"$so_proxy\";"\
"Acquire::https::Proxy \"$so_proxy\";" > /etc/apt/apt.conf.d/00-proxy.conf
fi
# Set global git proxy
printf '%s\n'\
"[http]"\
" proxy = ${so_proxy}" > /etc/gitconfig
}
setup_salt_master_dirs() {
@@ -2226,6 +2375,7 @@ set_progress_str() {
local percentage_input=$1
progress_bar_text=$2
export progress_bar_text
local nolog=$2
if (( "$percentage_input" >= "$percentage" )); then
percentage="$percentage_input"
@@ -2235,12 +2385,14 @@ set_progress_str() {
echo -e "$percentage_str"
if [[ -z $nolog ]]; then
info "Progressing ($percentage%): $progress_bar_text"
printf '%s\n' \
'----'\
"$percentage% - ${progress_bar_text^^}"\
"----" >> "$setup_log" 2>&1
# printf '%s\n' \
# '----'\
# "$percentage% - ${progress_bar_text^^}"\
# "----" >> "$setup_log" 2>&1
fi
}
set_ssh_cmds() {

98
setup/so-setup
View File

@@ -27,6 +27,8 @@ original_args=("$@")
cd "$(dirname "$0")" || exit 255
echo "Getting started..."
# Source the generic function libraries that are also used by the product after
# setup. These functions are intended to be reusable outside of the setup process.
source ../salt/common/tools/sbin/so-common
@@ -93,12 +95,23 @@ if ! [ -f $install_opt_file ]; then
analyze_system
fi
# Set up handler for setup to exit early (use `kill -SIGUSR1 "$setup_proc"; exit 1` in child scripts)
trap 'catch $LINENO' SIGUSR1
setup_proc="$$"
catch() {
info "Fatal error occurred at $1 in so-setup, failing setup."
grep --color=never "ERROR" "$setup_log" > "$error_log"
whiptail_setup_failed
exit 1
}
automated=no
function progress() {
local title='Security Onion Install'
progress() {
local title='Security Onion Setup'
local msg=${1:-'Please wait while installing...'}
if [ $automated == no ]; then
whiptail --title "$title" --gauge 'Please wait while installing...' 6 60 0 # append to text
whiptail --title "$title" --gauge "$msg" 6 70 0 # append to text
else
cat >> $setup_log 2>&1
fi
@@ -154,12 +167,9 @@ set_ssh_cmds $automated
local_sbin="$(pwd)/../salt/common/tools/sbin"
export PATH=$PATH:$local_sbin
installer_prereq_packages && detect_cloud
set_network_dev_status_list
set_palette >> $setup_log 2>&1
if [ "$OS" == ubuntu ]; then
update-alternatives --set newt-palette /etc/newt/palette.original >> $setup_log 2>&1
fi
# Kernel messages can overwrite whiptail screen #812
# https://github.com/Security-Onion-Solutions/securityonion/issues/812
@@ -192,19 +202,24 @@ if ! [[ -f $install_opt_file ]]; then
if [[ $setup_type == 'iso' ]] && [ "$automated" == no ]; then
whiptail_first_menu_iso
if [[ $option == "CONFIGURENETWORK" ]]; then
collect_hostname
network_init_whiptail
whiptail_management_interface_setup
network_init
printf '%s\n' \
"MNIC=$MNIC" \
"HOSTNAME=$HOSTNAME" > "$net_init_file"
set_main_ip >> $setup_log 2>&1
compare_main_nic_ip
reset_proxy
collect_proxy
[[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1
whiptail_net_setup_complete
else
whiptail_install_type
true
fi
else
whiptail_install_type
fi
whiptail_install_type
else
source $install_opt_file
fi
@@ -257,6 +272,10 @@ if [[ ( $is_manager || $is_import ) && $is_iso ]]; then
fi
fi
if [[ $is_manager || $is_import ]]; then
check_elastic_license
fi
if ! [[ -f $install_opt_file ]]; then
if [[ $is_manager && $is_sensor ]]; then
check_requirements "standalone"
@@ -273,25 +292,31 @@ if ! [[ -f $install_opt_file ]]; then
[[ -f $net_init_file ]] && whiptail_net_reinit && reinit_networking=true
if [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then
collect_hostname
network_init_whiptail
else
source "$net_init_file"
fi
if [[ $is_minion ]]; then
collect_mngr_hostname
fi
if [[ $is_minion ]] || [[ $reinit_networking ]] || [[ $is_iso ]] && ! [[ -f $net_init_file ]]; then
whiptail_management_interface_setup
fi
if [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then
network_init
fi
if [[ -n "$TURBO" ]]; then
use_turbo_proxy
set_main_ip >> $setup_log 2>&1
compare_main_nic_ip
if [[ $is_minion ]]; then
collect_mngr_hostname
fi
reset_proxy
if [[ -z $is_airgap ]]; then
collect_proxy
[[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1
fi
if [[ $is_minion ]] || [[ $reinit_networking ]] || [[ $is_iso ]] && ! [[ -f $net_init_file ]]; then
whiptail_management_interface_setup
fi
if [[ $is_minion ]]; then
@@ -310,6 +335,7 @@ if ! [[ -f $install_opt_file ]]; then
"HOSTNAME=$HOSTNAME" \
"MSRV=$MSRV" \
"MSRVIP=$MSRVIP" > "$install_opt_file"
[[ -n $so_proxy ]] && echo "so_proxy=$so_proxy" >> "$install_opt_file"
download_repo_tarball
exec bash /root/manager_setup/securityonion/setup/so-setup "${original_args[@]}"
fi
@@ -323,6 +349,22 @@ else
rm -rf $install_opt_file >> "$setup_log" 2>&1
fi
percentage=0
{
installer_progress_loop & # Run progress bar to 98 in ~8 minutes while waiting for package installs
progress_bg_proc=$!
installer_prereq_packages
install_success=$?
kill -9 "$progress_bg_proc"
wait "$progress_bg_proc" &> /dev/null # Kill just sends signal, redirect output of wait to catch stdout
if [[ $install_success -gt 0 ]]; then
echo "Could not install packages required for setup, exiting now." >> "$setup_log" 2>&1
kill -SIGUSR1 "$setup_proc"; exit 1
fi
} | progress '...'
detect_cloud
short_name=$(echo "$HOSTNAME" | awk -F. '{print $1}')
MINION_ID=$(echo "${short_name}_${install_type}" | tr '[:upper:]' '[:lower:]')
@@ -527,21 +569,10 @@ whiptail_make_changes
# From here on changes will be made.
echo "1" > /root/accept_changes
# Set up handler for setup to exit early (use `kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1` in child scripts)
trap 'catch $LINENO' SIGUSR1
catch() {
info "Fatal error occurred at $1 in so-setup, failing setup."
grep --color=never "ERROR" "$setup_log" > "$error_log"
whiptail_setup_failed
exit
}
# This block sets REDIRECTIT which is used by a function outside the below subshell
set_main_ip >> $setup_log 2>&1
compare_main_nic_ip
set_redirect >> $setup_log 2>&1
# Begin install
{
# Set initial percentage to 0
@@ -768,6 +799,9 @@ set_redirect >> $setup_log 2>&1
set_progress_str 70 "$(print_salt_state_apply 'kibana')"
salt-call state.apply -l info kibana >> $setup_log 2>&1
set_progress_str 70 "Setting up default Space in Kibana"
so-kibana-space-defaults >> $setup_log 2>&1
fi
if [[ "$PLAYBOOK" = 1 ]]; then

65
setup/so-whiptail
View File

@@ -588,6 +588,19 @@ whiptail_invalid_input() { # TODO: This should accept a list of arguments to spe
}
whiptail_invalid_proxy() {
[ -n "$TESTING" ] && return
local message
read -r -d '' message <<- EOM
Could not reach test url using proxy ${proxy_addr}.
Error was: ${proxy_test_err}
EOM
whiptail --title "Security Onion Setup" --yesno "$message" --yes-button "Enter Again" --no-button "Skip" 11 60
}
whiptail_invalid_string() {
[ -n "$TESTING" ] && return
@@ -1216,6 +1229,58 @@ whiptail_patch_schedule_select_hours() {
}
whiptail_proxy_ask() {
[ -n "$TESTING" ] && return
whiptail --title "Security Onion Setup" --yesno "Do you want to set a proxy server for this installation?" 7 60 --defaultno
}
whiptail_proxy_addr() {
[ -n "$TESTING" ] && return
local message
read -r -d '' message <<- EOM
Please input the proxy server you wish to use, including the URL prefix (ex: https://your.proxy.com:1234).
If your proxy requires a username and password do not include them in your input. Setup will ask for those values next.
EOM
proxy_addr=$(whiptail --title "Security Onion Setup" --inputbox "$message" 13 60 "$1" 3>&1 1>&2 2>&3)
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
}
whiptail_proxy_auth_ask() {
[ -n "$TESTING" ] && return
whiptail --title "Security Onion Setup" --yesno "Does your proxy require authentication?" 7 60
}
whiptail_proxy_auth_user() {
[ -n "$TESTING" ] && return
proxy_user=$(whiptail --title "Security Onion Setup" --inputbox "Please input the proxy user:" 8 60 "$1" 3>&1 1>&2 2>&3)
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
}
whiptail_proxy_auth_pass() {
local arg=$1
[ -n "$TESTING" ] && return
if [[ $arg != 'confirm' ]]; then
proxy_pass=$(whiptail --title "Security Onion Setup" --passwordbox "Please input the proxy password:" 8 60 3>&1 1>&2 2>&3)
else
proxy_pass_confirm=$(whiptail --title "Security Onion Setup" --passwordbox "Please confirm the proxy password:" 8 60 3>&1 1>&2 2>&3)
fi
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
}
whiptail_requirements_error() {
local requirement_needed=$1

BIN
sigs/securityonion-2.3.40.iso.sig Normal file
View File

Binary file not shown.