CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-06 17:22:49 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'remotes/origin/dev' into issue/90

Browse Source
This commit is contained in:
m0duspwnens
2020-03-24 17:28:46 -04:00
parent 7e6c70aff2 e4e598f0be
commit b96579f6d3
17 changed files with 917 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
exclude-list.txt
View File

@@ -1,2 +0,0 @@
salt/bro/files/local.bro
salt/bro/files/local.bro.community

37
salt/common/tools/sbin/soup Normal file
View File

@@ -0,0 +1,37 @@
#!/bin/bash
# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
clone_to_tmp() {
# TODO Need to add a air gap option
# Make a temp location for the files
rm -rf /tmp/soup
mkdir -p /tmp/soup
cd /tmp/soup
#git clone -b dev https://github.com/Security-Onion-Solutions/securityonion-saltstack.git
git clone https://github.com/Security-Onion-Solutions/securityonion-saltstack.git
}
# Prompt the user that this requires internets
clone_to_tmp
cd /tmp/soup/securityonion-saltstack/update
chmod +x soup
./soup

7
salt/elasticsearch/files/ingest/ossec.alert
View File

@@ -8,14 +8,15 @@
{ "rename": { "field": "message2.decoder", "target_field": "decoder", "ignore_missing": true } },
{ "rename": { "field": "message2.full_log", "target_field": "log.full", "ignore_missing": true } },
{ "rename": { "field": "message2.id", "target_field": "log.id.id", "ignore_missing": true } },
{ "rename": { "field": "message2.location", "target_field": "location", "ignore_missing": true } },
{ "rename": { "field": "message2.location", "target_field": "log.location", "ignore_missing": true } },
{ "rename": { "field": "message2.manager", "target_field": "manager", "ignore_missing": true } },
{ "rename": { "field": "message2.predecoder", "target_field": "predecoder", "ignore_missing": true } },
{ "rename": { "field": "message2.timestamp", "target_field": "timestamp", "ignore_missing": true } },
{ "rename": { "field": "message2.timestamp", "target_field": "event.timestamp", "ignore_missing": true } },
{ "rename": { "field": "message2.previous_log", "target_field": "log.previous_log", "ignore_missing": true } },
{ "rename": { "field": "message2.previous_output", "target_field": "log.previous_output", "ignore_missing": true } },
{ "rename": { "field": "message2.rule", "target_field": "rule", "ignore_missing": true } },
{ "rename": { "field": "data.command", "target_field": "command", "ignore_missing": true } },
{ "rename": { "field": "message2.syscheck", "target_field": "host.syscheck", "ignore_missing": true } },
{ "rename": { "field": "data.command", "target_field": "process.command_line", "ignore_missing": true } },
{ "rename": { "field": "data.dstip", "target_field": "destination.ip", "ignore_missing": true } },
{ "rename": { "field": "data.dstport", "target_field": "destination.port", "ignore_missing": true } },
{ "rename": { "field": "data.dstuser", "target_field": "user.escalated", "ignore_missing": true } },

3
salt/elasticsearch/files/ingest/zeek.weird
View File

@@ -6,6 +6,9 @@
{ "rename": { "field": "message2.addl", "target_field": "weird.additional_info", "ignore_missing": true } },
{ "rename": { "field": "message2.notice", "target_field": "weird.notice", "ignore_missing": true } },
{ "rename": { "field": "message2.peer", "target_field": "weird.peer", "ignore_missing": true } },
{ "rename": { "field": "message2.p", "target_field": "weird.p", "ignore_missing": true } },
{ "rename": { "field": "message2.dst", "target_field": "destination.ip", "ignore_missing": true } },
{ "rename": { "field": "message2.src", "target_field": "source.ip", "ignore_missing": true } },
{ "pipeline": { "name": "zeek.common" } }
]
}

34
salt/kibana/bin/so-kibana-config-load Normal file
View File

@@ -0,0 +1,34 @@
#!/bin/bash
KIBANA_VERSION="7.6.1"
MAX_WAIT=60
# Check to see if Kibana is available
wait_step=0
until curl -s -XGET http://localhost:5601 > /dev/null ; do
wait_step=$(( ${wait_step} + 1 ))
echo "Waiting on Kibana...Attempt #$wait_step"
if [ ${wait_step} -gt ${MAX_WAIT} ]; then
echo "ERROR: Kibana not available for more than ${MAX_WAIT} seconds."
exit 5
fi
sleep 1s;
done
# Sleep additional JIC server is not ready
sleep 30s
# Load config
curl -X PUT "localhost:5601/api/saved_objects/config/$KIBANA_VERSION" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'
{ "attributes":
{
"defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29",
"defaultRoute":"/app/kibana#/dashboard/a8411b30-6d03-11ea-b301-3d6c35840645",
"discover:sampleSize":"100",
"dashboard:defaultDarkTheme":true,
"theme:darkMode":true,
"timepicker:timeDefaults":"{\n \"from\": \"now-24h\",\n \"to\": \"now\",\n \"mode\": \"quick\"\n}"
}
}'
# Load saved objects
curl -X POST "localhost:5601/api/saved_objects/_import" -H "kbn-xsrf: true" --form file=@/opt/so/saltstack/salt/kibana/saved_objects.ndjson

3
salt/kibana/etc/config.json
View File

@@ -1,6 +1,7 @@
{ "attributes":
{
"defaultIndex": "*:so-*",
"defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29",
"defaultRoute":"/app/kibana#/dashboard/a8411b30-6d03-11ea-b301-3d6c35840645",
"discover:sampleSize":"100",
"dashboard:defaultDarkTheme":true,
"theme:darkMode":true,

1
salt/kibana/etc/kibana.yml
View File

@@ -11,3 +11,4 @@ elasticsearch.hosts: [ "http://{{ ES }}:9200" ]
#xpack.monitoring.ui.container.elasticsearch.enabled: true
elasticsearch.requestTimeout: 90000
logging.dest: /var/log/kibana/kibana.log
telemetry.enabled: false

19
salt/kibana/init.sls
View File

@@ -1,4 +1,4 @@
{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.1.4') %}
{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.2.1') %}
{% set MASTER = salt['grains.get']('master') %}
{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
{% if FEATURES %}
@@ -59,6 +59,8 @@ synckibanacustom:
- user: 932
- group: 939
# File.Recurse for custom saved dashboards
# Start the kibana docker
so-kibana:
docker_container.running:
@@ -66,7 +68,6 @@ so-kibana:
- hostname: kibana
- user: kibana
- environment:
- KIBANA_DEFAULTAPPID=dashboard/94b52620-342a-11e7-9d52-4f090484f59e
- ELASTICSEARCH_HOST={{ MASTER }}
- ELASTICSEARCH_PORT=9200
- MASTER={{ MASTER }}
@@ -77,3 +78,17 @@ so-kibana:
- /sys/fs/cgroup:/sys/fs/cgroup:ro
- port_bindings:
- 0.0.0.0:5601:5601
so-kibana-config-load:
cmd.script:
- shell: /bin/bash
- runas: socore
- source: salt://kibana/bin/so-kibana-config-load
# Keep the setting correct
#KibanaHappy:
# cmd.script:
# - shell: /bin/bash
# - runas: socore
# - source: salt://kibana/bin/keepkibanahappy.sh
# - template: jinja

507
salt/kibana/saved_objects.ndjson Normal file
View File

File diff suppressed because one or more lines are too long

8
salt/logstash/pipelines/templates/so/so-common-template.json
View File

@@ -192,6 +192,14 @@
"type":"object",
"dynamic": true
},
"message":{
"type":"text",
"fields":{
"keyword":{
"type":"keyword"
}
}
},
"modbus":{
"type":"object",
"dynamic": true

16
salt/registry/init.sls
View File

@@ -27,15 +27,15 @@ dockerregistryconf:
- source: salt://registry/etc/config.yml
# Copy the registry script
dockerregistrybuild:
file.managed:
- name: /opt/so/conf/docker-registry/so-buildregistry
- source: salt://registry/bin/so-buildregistry
- mode: 755
#dockerregistrybuild:
# file.managed:
# - name: /opt/so/conf/docker-registry/so-buildregistry
# - source: salt://registry/bin/so-buildregistry
# - mode: 755
dockerexpandregistry:
cmd.run:
- name: /opt/so/conf/docker-registry/so-buildregistry
#dockerexpandregistry:
# cmd.run:
# - name: /opt/so/conf/docker-registry/so-buildregistry
# Install the registry container
so-dockerregistry:

14
setup/so-functions
View File

@@ -17,6 +17,7 @@
SCRIPTDIR=$(dirname "$0")
source $SCRIPTDIR/so-whiptail
SOVERSION=1.2.1
accept_salt_key_local() {
echo "Accept the key locally on the master" >> $SETUPLOG 2>&1
@@ -460,7 +461,7 @@ docker_registry() {
}
docker_seed_registry() {
VERSION="HH1.2.1"
VERSION="HH$SOVERSION"
if [ $INSTALLTYPE != 'HELIXSENSOR' ]; then
TRUSTED_CONTAINERS=( \
"so-acng:$VERSION" \
@@ -743,7 +744,7 @@ master_static() {
touch /opt/so/saltstack/pillar/static.sls
echo "static:" > /opt/so/saltstack/pillar/static.sls
echo " soversion: HH1.2.1" >> /opt/so/saltstack/pillar/static.sls
echo " soversion: HH$SOVERSION" >> /opt/so/saltstack/pillar/static.sls
echo " hnmaster: $HNMASTER" >> /opt/so/saltstack/pillar/static.sls
echo " ntpserver: $NTPSERVER" >> /opt/so/saltstack/pillar/static.sls
echo " proxy: $PROXY" >> /opt/so/saltstack/pillar/static.sls
@@ -1280,8 +1281,8 @@ set_hostname() {
echo "::1 localhost localhost.localdomain localhost6 localhost6.localdomain6" >> /etc/hosts
echo $HOSTNAME > /etc/hostname
HOSTNAME=$(cat /etc/hostname)
if [ $INSTALLTYPE != 'MASTER' ] || [ $INSTALLTYPE != 'EVAL' ] || [ $INSTALLTYPE == 'HELIXSENSOR' ] || [ $INSTALLTYPE == 'MASTERSEARCH' ]; then
if [[ $TESTHOST = *"not found"* ]] || [[ $TESTHOST = *"connection timed out"* ]]; then
if [ $INSTALLTYPE != 'MASTER' || $INSTALLTYPE != 'EVAL' || $INSTALLTYPE == 'HELIXSENSOR' || $INSTALLTYPE == 'MASTERSEARCH' ]; then
if [[ $TESTHOST = *"not found"* ]] || [ -z $TESTHOST ] || [[ $TESTHOST = *"connection timed out"* ]]; then
if ! grep -q $MSRVIP /etc/hosts; then
echo "$MSRVIP $MSRV" >> /etc/hosts
fi
@@ -1412,6 +1413,11 @@ set_updates() {
fi
}
set_version() {
# Drop a file with the current version
echo "$SOVERSION" > /etc/soversion
}
update_sudoers() {
if ! grep -qE '^socore\ ALL=\(ALL\)\ NOPASSWD:(\/usr\/bin\/salt\-key|\/opt\/so\/saltstack)' /etc/sudoers; then

3
setup/so-setup
View File

@@ -159,6 +159,7 @@ if (whiptail_you_sure) ; then
calculate_useable_cores
whiptail_make_changes
set_hostname
set_version
clear_master
mkdir -p /nsm
get_filesystem_root
@@ -302,6 +303,7 @@ if (whiptail_you_sure) ; then
# Last Chance to back out
whiptail_make_changes
set_hostname
set_version
generate_passwords
auth_pillar
clear_master
@@ -570,6 +572,7 @@ if (whiptail_you_sure) ; then
fi
whiptail_make_changes
set_hostname
set_version
generate_passwords
auth_pillar
clear_master

2
so-setup-network
View File

@@ -1,6 +1,6 @@
#!/bin/bash
# Copyright 2014,2015,2016,2017,2018,2019 Security Onion Solutions, LLC
# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by

15
updatemaster
View File

@@ -1,15 +0,0 @@
#!/bin/bash
# Clone github
mkdir /tmp/sogh
cd /tmp/sogh
#git clone -b dev https://github.com/Security-Onion-Solutions/securityonion-saltstack.git
git clone https://github.com/Security-Onion-Solutions/securityonion-saltstack.git
cd securityonion-saltstack
rsync -a --exclude-from 'exclude-list.txt' salt /opt/so/saltstack/
chown -R socore:socore /opt/so/saltstack/salt
chmod 755 /opt/so/saltstack/pillar/firewall/addfirewall.sh
cd ~
rm -rf /tmp/sogh
# Run so-elastic-download here and call this soup with some magic
salt-call state.highstate

207
upgrade/so-update-functions Normal file
View File

@@ -0,0 +1,207 @@
#!/bin/bash
# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# Set the new SO Version
UPDATEVERSION=1.2.1
BUILD=HH
#Determine the current install version
if [ -f /etc/soversion ]; then
OLDVERSION=$(cat /etc/soversion)
else
OLDVERSION=1.1.4
fi
# Use the hostname
HOSTNAME=$(hostname)
# List all the containers
if [ $MASTERCHECK != 'so-helix' ]; then
TRUSTED_CONTAINERS=( \
"so-acng:$BUILD$UPDATEVERSION" \
"so-auth-api:$BUILD$UPDATEVERSION" \
"so-auth-ui:$BUILD$UPDATEVERSION" \
"so-core:$BUILD$UPDATEVERSION" \
"so-thehive-cortex:$BUILD$UPDATEVERSION" \
"so-curator:$BUILD$UPDATEVERSION" \
"so-domainstats:$BUILD$UPDATEVERSION" \
"so-elastalert:$BUILD$UPDATEVERSION" \
"so-elasticsearch:$BUILD$UPDATEVERSION" \
"so-filebeat:$BUILD$UPDATEVERSION" \
"so-fleet:$BUILD$UPDATEVERSION" \
"so-fleet-launcher:$BUILD$UPDATEVERSION" \
"so-freqserver:$BUILD$UPDATEVERSION" \
"so-grafana:$BUILD$UPDATEVERSION" \
"so-idstools:$BUILD$UPDATEVERSION" \
"so-influxdb:$BUILD$UPDATEVERSION" \
"so-kibana:$BUILD$UPDATEVERSION" \
"so-logstash:$BUILD$UPDATEVERSION" \
"so-mysql:$BUILD$UPDATEVERSION" \
"so-navigator:$BUILD$UPDATEVERSION" \
"so-playbook:$BUILD$UPDATEVERSION" \
"so-redis:$BUILD$UPDATEVERSION" \
"so-sensoroni:$BUILD$UPDATEVERSION" \
"so-soctopus:$BUILD$UPDATEVERSION" \
"so-steno:$BUILD$UPDATEVERSION" \
"so-strelka:$BUILD$UPDATEVERSION" \
"so-suricata:$BUILD$UPDATEVERSION" \
"so-telegraf:$BUILD$UPDATEVERSION" \
"so-thehive:$BUILD$UPDATEVERSION" \
"so-thehive-es:$BUILD$UPDATEVERSION" \
"so-wazuh:$BUILD$UPDATEVERSION" \
"so-zeek:$BUILD$UPDATEVERSION" )
else
TRUSTED_CONTAINERS=( \
"so-core:$BUILD$UPDATEVERSION" \
"so-filebeat:$BUILD$UPDATEVERSION" \
"so-idstools:$BUILD$UPDATEVERSION" \
"so-logstash:$BUILD$UPDATEVERSION" \
"so-redis:$BUILD$UPDATEVERSION" \
"so-sensoroni:$BUILD$UPDATEVERSION" \
"so-steno:$BUILD$UPDATEVERSION" \
"so-suricata:$BUILD$UPDATEVERSION" \
"so-telegraf:$BUILD$UPDATEVERSION" \
"so-zeek:$BUILD$UPDATEVERSION" )
fi
clone_to_tmp() {
# TODO Need to add a air gap option
# Make a temp location for the files
mkdir /tmp/sogh
cd /tmp/sogh
#git clone -b dev https://github.com/Security-Onion-Solutions/securityonion-saltstack.git
git clone https://github.com/Security-Onion-Solutions/securityonion-saltstack.git
cd /tmp
}
copy_new_files() {
# Copy new files over to the salt dir
cd /tmp/sogh/securityonion-saltstack
rsync -a --exclude-from 'exclude-list.txt' salt /opt/so/saltstack/
chown -R socore:socore /opt/so/saltstack/salt
chmod 755 /opt/so/saltstack/pillar/firewall/addfirewall.sh
cd /tmp
}
detect_os() {
# Detect Base OS
echo "Detecting Base OS" >> $UPDATELOG 2>&1
if [ -f /etc/redhat-release ]; then
OS=centos
if grep -q "CentOS Linux release 7" /etc/redhat-release; then
OSVER=7
elif grep -q "CentOS Linux release 8" /etc/redhat-release; then
OSVER=8
echo "We currently do not support CentOS $OSVER but we are working on it!"
exit
else
echo "We do not support the version of CentOS you are trying to use"
exit
fi
elif [ -f /etc/os-release ]; then
OS=ubuntu
if grep -q "UBUNTU_CODENAME=bionic" /etc/os-release; then
OSVER=bionic
elif grep -q "UBUNTU_CODENAME=xenial" /etc/os-release; then
OSVER=xenial
else
echo "We do not support your current version of Ubuntu"
exit
fi
else
echo "We were unable to determine if you are using a supported OS." >> $UPDATELOG 2>&1
exit
fi
echo "Found OS: $OS $OSVER" >> $UPDATELOG 2>&1
}
master_check() {
# Check to see if this is a master
MASTERCHECK=$(cat /etc/salt/grains | grep role | awk '{print $2}')
if [ $MASTERCHECK == 'so-eval' OR $MASTERCHECK == 'so-master' OR $MASTERCHECK == 'so-mastersearch' ]; then
echo "This is a master. We can proceed"
else
echo "Please run soup on the master. The master controls all updates."
exit
}
salt_highstate() {
salt-call state.highstate
}
update_held_packages() {
if [ $OS == "centos" ]
SALTVER=2019.2.3
DOCKERVER=
yum -y --disableexcludes=all update salt-$SALTVER
yum -y --disableexcludes=all update docker-ce-$DOCKERVER
else
SALTVER=2019.2.3+ds-1
DOCKERVER=5:19.03.8~3-0~ubuntu-xenial
fi
}
update_all_packages() {
# Update all the things based on OS
if [ $OS == "centos" ]; then
yum -y update
else
apt -y update && apt -y upgrade
fi
}
update_docker_containers() {
# Download the containers from the interwebs
for i in "${TRUSTED_CONTAINERS[@]}"
do
# Pull down the trusted docker image
echo "Downloading $i"
docker pull --disable-content-trust=false docker.io/soshybridhunter/$i
# Tag it with the new registry destination
docker tag soshybridhunter/$i $HOSTNAME:5000/soshybridhunter/$i
docker push $HOSTNAME:5000/soshybridhunter/$i
done
for i in "${TRUSTED_CONTAINERS[@]}"
do
echo "Removing $i locally"
docker rmi soshybridhunter/$i
done
}
update_hh_version() {
# Change the version number in the static pillar
}

27
upgrade/soup Normal file
View File

@@ -0,0 +1,27 @@
#!/bin/bash
# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
SCRIPTDIR=$(dirname "$0")
source $SCRIPTDIR/so-update-functions
# Update Packages
master_check
update_all_packages
update_held_packages