diff --git a/salt/common/tools/sbin/so-elastic-restart b/salt/common/tools/sbin/so-elastic-restart
new file mode 100755
index 000000000..0e3c5937d
--- /dev/null
+++ b/salt/common/tools/sbin/so-elastic-restart
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+. /usr/sbin/so-common
+
+
+{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-node', 'so-import']%}
+/usr/sbin/so-restart elasticsearch $1
+{%- endif %}
+
+{%- if grains['role'] in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%}
+/usr/sbin/so-restart kibana $1
+{%- endif %}
+
+{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-node']%}
+/usr/sbin/so-restart logstash $1
+{%- endif %}
+
+{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-node', 'so-sensor']%}
+/usr/sbin/so-restart filebeat $1
+{%- endif %}
+
+{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-node']%}
+/usr/sbin/so-restart curator $1
+{%- endif %}
+
+{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%}
+/usr/sbin/so-restart elastalert $1
+{%- endif %}
diff --git a/salt/common/tools/sbin/so-elastic-start b/salt/common/tools/sbin/so-elastic-start
new file mode 100755
index 000000000..51657ff54
--- /dev/null
+++ b/salt/common/tools/sbin/so-elastic-start
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+. /usr/sbin/so-common
+
+
+{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-node', 'so-import']%}
+/usr/sbin/so-start elasticsearch $1
+{%- endif %}
+
+{%- if grains['role'] in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%}
+/usr/sbin/so-start kibana $1
+{%- endif %}
+
+{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-node']%}
+/usr/sbin/so-start logstash $1
+{%- endif %}
+
+{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-node', 'so-sensor']%}
+/usr/sbin/so-start filebeat $1
+{%- endif %}
+
+{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-node']%}
+/usr/sbin/so-start curator $1
+{%- endif %}
+
+{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%}
+/usr/sbin/so-start elastalert $1
+{%- endif %}
diff --git a/salt/common/tools/sbin/so-elastic-stop b/salt/common/tools/sbin/so-elastic-stop
new file mode 100755
index 000000000..2f6c46082
--- /dev/null
+++ b/salt/common/tools/sbin/so-elastic-stop
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+. /usr/sbin/so-common
+
+
+{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-node', 'so-import']%}
+/usr/sbin/so-stop elasticsearch $1
+{%- endif %}
+
+{%- if grains['role'] in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%}
+/usr/sbin/so-stop kibana $1
+{%- endif %}
+
+{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-node']%}
+/usr/sbin/so-stop logstash $1
+{%- endif %}
+
+{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-node', 'so-sensor']%}
+/usr/sbin/so-stop filebeat $1
+{%- endif %}
+
+{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-node']%}
+/usr/sbin/so-stop curator $1
+{%- endif %}
+
+{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%}
+/usr/sbin/so-stop elastalert $1
+{%- endif %}
diff --git a/salt/common/tools/sbin/so-salt-minion-check b/salt/common/tools/sbin/so-salt-minion-check
old mode 100644
new mode 100755
diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index 27439a137..da534281e 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -155,6 +155,13 @@ copy_new_files() {
cd /tmp
}
+generate_and_clean_tarballs() {
+ local new_version
+ new_version=$(cat $UPDATE_DIR/VERSION)
+ tar -cxf "/opt/so/repo/$new_version.tar.gz" "$UPDATE_DIR"
+ find "/opt/so/repo" -type f -not -name "$new_version.tar.gz" -exec rm -rf {} \;
+}
+
highstate() {
# Run a highstate.
salt-call state.highstate -l info queue=True
@@ -417,6 +424,8 @@ else
echo "Cloning Security Onion github repo into $UPDATE_DIR."
clone_to_tmp
fi
+echo "Generating new repo archive"
+generate_and_clean_tarballs
if [ -f /usr/sbin/so-image-common ]; then
. /usr/sbin/so-image-common
else
diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml
index acad465d1..18d1c9c81 100644
--- a/salt/elasticsearch/files/elasticsearch.yml
+++ b/salt/elasticsearch/files/elasticsearch.yml
@@ -44,3 +44,4 @@ cluster.routing.allocation.disk.watermark.flood_stage: 98%
node.attr.box_type: {{ NODE_ROUTE_TYPE }}
node.name: {{ ESCLUSTERNAME }}
script.max_compilations_rate: 1000/1m
+indices.query.bool.max_clause_count: 1500
diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml
index cb2de370c..30a6117aa 100644
--- a/salt/firewall/assigned_hostgroups.map.yaml
+++ b/salt/firewall/assigned_hostgroups.map.yaml
@@ -1,6 +1,7 @@
{% set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') %}
{% import_yaml 'firewall/portgroups.yaml' as portgroups %}
{% set portgroups = portgroups.firewall.aliases.ports %}
+{% set TRUE_CLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %}
role:
eval:
@@ -42,6 +43,11 @@ role:
- {{ portgroups.redis }}
- {{ portgroups.minio }}
- {{ portgroups.elasticsearch_node }}
+ heavy_node:
+ portgroups:
+ - {{ portgroups.redis }}
+ - {{ portgroups.minio }}
+ - {{ portgroups.elasticsearch_node }}
self:
portgroups:
- {{ portgroups.syslog}}
@@ -135,6 +141,12 @@ role:
- {{ portgroups.minio }}
- {{ portgroups.elasticsearch_node }}
- {{ portgroups.beats_5644 }}
+ heavy_node:
+ portgroups:
+ - {{ portgroups.redis }}
+ - {{ portgroups.minio }}
+ - {{ portgroups.elasticsearch_node }}
+ - {{ portgroups.beats_5644 }}
self:
portgroups:
- {{ portgroups.syslog}}
@@ -219,6 +231,11 @@ role:
- {{ portgroups.redis }}
- {{ portgroups.minio }}
- {{ portgroups.elasticsearch_node }}
+ heavy_node:
+ portgroups:
+ - {{ portgroups.redis }}
+ - {{ portgroups.minio }}
+ - {{ portgroups.elasticsearch_node }}
self:
portgroups:
- {{ portgroups.syslog}}
@@ -303,6 +320,11 @@ role:
- {{ portgroups.redis }}
- {{ portgroups.minio }}
- {{ portgroups.elasticsearch_node }}
+ heavy_node:
+ portgroups:
+ - {{ portgroups.redis }}
+ - {{ portgroups.minio }}
+ - {{ portgroups.elasticsearch_node }}
self:
portgroups:
- {{ portgroups.syslog}}
@@ -425,6 +447,11 @@ role:
elasticsearch_rest:
portgroups:
- {{ portgroups.elasticsearch_rest }}
+ {% if TRUE_CLUSTER %}
+ search_node:
+ portgroups:
+ - {{ portgroups.elasticsearch_node }}
+ {% endif %}
self:
portgroups:
- {{ portgroups.syslog}}
diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json
index cc5be34ea..55b928ef0 100644
--- a/salt/sensoroni/files/sensoroni.json
+++ b/salt/sensoroni/files/sensoroni.json
@@ -1,12 +1,16 @@
{% set URLBASE = salt['pillar.get']('global:url_base') -%}
+{% set DESCRIPTION = salt['pillar.get']('sensoroni:node_description') -%}
+{% set ADDRESS = salt['pillar.get']('sensoroni:node_address') -%}
{% set SENSORONIKEY = salt['pillar.get']('global:sensoronikey', '') -%}
-{% set CHECKININTERVALMS = salt['pillar.get']('sensoroni:sensor_checkin_interval_ms', 10000) -%}
+{% set CHECKININTERVALMS = salt['pillar.get']('sensoroni:node_checkin_interval_ms', 10000) -%}
{% set STENOENABLED = salt['pillar.get']('steno:enabled', False) -%}
{
"logFilename": "/opt/sensoroni/logs/sensoroni.log",
"logLevel":"info",
"agent": {
"role": "{{ grains.role }}",
+ "description": "{{ DESCRIPTION }}",
+ "address": "{{ ADDRESS }}",
"pollIntervalMs": {{ CHECKININTERVALMS if CHECKININTERVALMS else 10000 }},
"serverUrl": "https://{{ URLBASE }}/sensoroniagents",
"verifyCert": false,
diff --git a/salt/soc/files/soc/alerts.actions.json b/salt/soc/files/soc/alerts.actions.json
index b825c0131..46c4ea68d 100644
--- a/salt/soc/files/soc/alerts.actions.json
+++ b/salt/soc/files/soc/alerts.actions.json
@@ -1,6 +1,29 @@
[
- { "name": "actionHunt", "description": "actionHuntHelp", "icon": "fa-crosshairs", "link": "/#/hunt?q=\"{value}\" | groupby event.module event.dataset", "target": "" },
- { "name": "actionPcap", "description": "actionPcapHelp", "icon": "fa-stream", "link": "/joblookup?esid={eventId}", "target": "" },
- { "name": "actionGoogle", "description": "actionGoogleHelp", "icon": "fab fa-google", "link": "https://www.google.com/search?q={value}", "target": "_blank" },
- { "name": "actionVirusTotal", "description": "actionVirusTotalHelp", "icon": "fa-external-link-alt", "link": "https://www.virustotal.com/gui/search/{value}", "target": "_blank" }
+ { "name": "actionHunt", "description": "actionHuntHelp", "icon": "fa-crosshairs", "target": "",
+ "links": [
+ "/#/hunt?q=\"{value}\" | groupby event.module event.dataset"
+ ]},
+ { "name": "actionCorrelate", "description": "actionCorrelateHelp", "icon": "fab fa-searchengin", "target": "",
+ "links": [
+ "/#/hunt?q=\"{:log.id.fuid}\" OR \"{:log.id.uid}\" OR \"{:network.community_id}\" | groupby event.module event.dataset",
+ "/#/hunt?q=\"{:log.id.fuid}\" OR \"{:log.id.uid}\" | groupby event.module event.dataset",
+ "/#/hunt?q=\"{:log.id.fuid}\" OR \"{:network.community_id}\" | groupby event.module event.dataset",
+ "/#/hunt?q=\"{:log.id.uid}\" OR \"{:network.community_id}\" | groupby event.module event.dataset",
+ "/#/hunt?q=\"{:log.id.fuid}\" | groupby event.module event.dataset",
+ "/#/hunt?q=\"{:log.id.uid}\" | groupby event.module event.dataset",
+ "/#/hunt?q=\"{:network.community_id}\" | groupby event.module event.dataset"
+ ]},
+ { "name": "actionPcap", "description": "actionPcapHelp", "icon": "fa-stream", "target": "",
+ "links": [
+ "/joblookup?esid={:soc_id}",
+ "/joblookup?ncid={:network.community_id}"
+ ]},
+ { "name": "actionGoogle", "description": "actionGoogleHelp", "icon": "fab fa-google", "target": "_blank",
+ "links": [
+ "https://www.google.com/search?q={value}"
+ ]},
+ { "name": "actionVirusTotal", "description": "actionVirusTotalHelp", "icon": "fa-external-link-alt", "target": "_blank",
+ "links": [
+ "https://www.virustotal.com/gui/search/{value}"
+ ]}
]
\ No newline at end of file
diff --git a/salt/soc/files/soc/hunt.actions.json b/salt/soc/files/soc/hunt.actions.json
index b825c0131..46c4ea68d 100644
--- a/salt/soc/files/soc/hunt.actions.json
+++ b/salt/soc/files/soc/hunt.actions.json
@@ -1,6 +1,29 @@
[
- { "name": "actionHunt", "description": "actionHuntHelp", "icon": "fa-crosshairs", "link": "/#/hunt?q=\"{value}\" | groupby event.module event.dataset", "target": "" },
- { "name": "actionPcap", "description": "actionPcapHelp", "icon": "fa-stream", "link": "/joblookup?esid={eventId}", "target": "" },
- { "name": "actionGoogle", "description": "actionGoogleHelp", "icon": "fab fa-google", "link": "https://www.google.com/search?q={value}", "target": "_blank" },
- { "name": "actionVirusTotal", "description": "actionVirusTotalHelp", "icon": "fa-external-link-alt", "link": "https://www.virustotal.com/gui/search/{value}", "target": "_blank" }
+ { "name": "actionHunt", "description": "actionHuntHelp", "icon": "fa-crosshairs", "target": "",
+ "links": [
+ "/#/hunt?q=\"{value}\" | groupby event.module event.dataset"
+ ]},
+ { "name": "actionCorrelate", "description": "actionCorrelateHelp", "icon": "fab fa-searchengin", "target": "",
+ "links": [
+ "/#/hunt?q=\"{:log.id.fuid}\" OR \"{:log.id.uid}\" OR \"{:network.community_id}\" | groupby event.module event.dataset",
+ "/#/hunt?q=\"{:log.id.fuid}\" OR \"{:log.id.uid}\" | groupby event.module event.dataset",
+ "/#/hunt?q=\"{:log.id.fuid}\" OR \"{:network.community_id}\" | groupby event.module event.dataset",
+ "/#/hunt?q=\"{:log.id.uid}\" OR \"{:network.community_id}\" | groupby event.module event.dataset",
+ "/#/hunt?q=\"{:log.id.fuid}\" | groupby event.module event.dataset",
+ "/#/hunt?q=\"{:log.id.uid}\" | groupby event.module event.dataset",
+ "/#/hunt?q=\"{:network.community_id}\" | groupby event.module event.dataset"
+ ]},
+ { "name": "actionPcap", "description": "actionPcapHelp", "icon": "fa-stream", "target": "",
+ "links": [
+ "/joblookup?esid={:soc_id}",
+ "/joblookup?ncid={:network.community_id}"
+ ]},
+ { "name": "actionGoogle", "description": "actionGoogleHelp", "icon": "fab fa-google", "target": "_blank",
+ "links": [
+ "https://www.google.com/search?q={value}"
+ ]},
+ { "name": "actionVirusTotal", "description": "actionVirusTotalHelp", "icon": "fa-external-link-alt", "target": "_blank",
+ "links": [
+ "https://www.virustotal.com/gui/search/{value}"
+ ]}
]
\ No newline at end of file
diff --git a/salt/strelka/files/backend/backend.yaml b/salt/strelka/files/backend/backend.yaml
index 53c29e3fa..db6ce0560 100644
--- a/salt/strelka/files/backend/backend.yaml
+++ b/salt/strelka/files/backend/backend.yaml
@@ -6,8 +6,8 @@
{%- endif -%}
logging_cfg: '/etc/strelka/logging.yaml'
limits:
- max_files: 5000
- time_to_live: 900
+ max_files: 0
+ time_to_live: 0
max_depth: 15
distribution: 600
scanner: 150
@@ -215,14 +215,6 @@ scanners:
priority: 5
options:
tmp_directory: '/dev/shm/'
- 'ScanMmbot':
- - positive:
- flavors:
- - 'vb_file'
- - 'vbscript'
- priority: 5
- options:
- server: 'strelka_mmrpc_1:33907'
'ScanOcr':
- positive:
flavors:
diff --git a/setup/so-functions b/setup/so-functions
index 67cbb7c24..cd9b63ce2 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -518,6 +518,14 @@ check_requirements() {
fi
}
+compare_versions() {
+ manager_ver=$(ssh -i /root/.ssh/so.key soremote@"$MSRV" cat /etc/soversion)
+ export manager_ver
+
+ [[ "$manager_ver" == "$SOVERSION" ]]
+ return
+}
+
configure_network_sensor() {
echo "Setting up sensor interface" >> "$setup_log" 2>&1
local nic_error=0
@@ -771,7 +779,7 @@ disable_auto_start() {
logCmd "crontab -u $INSTALLUSERNAME -r"
fi
- if grep -q so-setup /home/$INSTALLUSERNAME/.bash_profile; then
+ if grep -s -q so-setup /home/$INSTALLUSERNAME/.bash_profile; then
# Truncate last line of the bash profile
info "Removing auto-run of setup from bash profile"
sed -i '$ d' /home/$INSTALLUSERNAME/.bash_profile >> "$setup_log" 2>&1
@@ -913,6 +921,18 @@ docker_seed_registry() {
}
+download_repo_tarball() {
+ scp -i /root/.ssh/so.key soremote@"$MSRV":/opt/so/repo/"$manager_ver".tar.gz /root/manager_setup
+
+ # Fail if the file doesn't download
+ if ! [ -f /root/manager_setup/"$manager_ver".tar.gz ]; then
+ kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
+ fi
+
+ tar -xzf /root/manager_setup/"$manager_ver".tar.gz -C /root/manager_setup/securityonion
+ rm -rf /root/manager_setup/"$manager_ver".tar.gz
+}
+
fireeye_pillar() {
local fireeye_pillar_path=$local_salt_dir/pillar/fireeye
@@ -972,6 +992,10 @@ generate_passwords(){
KRATOSKEY=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 20 | head -n 1)
}
+generate_repo_tarball() {
+ tar -czf /opt/so/repo/"$SOVERSION".tar.gz ../.
+}
+
get_redirect() {
whiptail_set_redirect
if [ "$REDIRECTINFO" = "OTHER" ]; then
@@ -1007,18 +1031,22 @@ host_pillar() {
}
install_cleanup() {
- echo "Installer removing the following files:"
- ls -lR "$temp_install_dir"
+ if [ -f "$temp_install_dir" ]; then
+ echo "Installer removing the following files:"
+ ls -lR "$temp_install_dir"
- # Clean up after ourselves
- rm -rf "$temp_install_dir"
+ # Clean up after ourselves
+ rm -rf "$temp_install_dir"
+ fi
# All cleanup prior to this statement must be compatible with automated testing. Cleanup
# that will disrupt automated tests should be placed beneath this statement.
[ -n "$TESTING" ] && return
# If Mysql is running stop it
- /usr/sbin/so-mysql-stop
+ if docker ps --format "{{.Names}}" 2>&1 | grep -q "so-mysql"; then
+ /usr/sbin/so-mysql-stop
+ fi
if [[ $setup_type == 'iso' ]]; then
info "Removing so-setup permission entry from sudoers file"
@@ -1103,10 +1131,10 @@ manager_pillar() {
manager_global() {
local global_pillar="$local_salt_dir/pillar/global.sls"
- if [ -z "$SENSOR_CHECKIN_INTERVAL_MS" ]; then
- SENSOR_CHECKIN_INTERVAL_MS=10000
+ if [ -z "$NODE_CHECKIN_INTERVAL_MS" ]; then
+ NODE_CHECKIN_INTERVAL_MS=10000
if [ "$install_type" = 'EVAL' ] || [ "$install_type" = 'STANDALONE' ] || [ "$install_type" = 'IMPORT' ]; then
- SENSOR_CHECKIN_INTERVAL_MS=1000
+ NODE_CHECKIN_INTERVAL_MS=1000
fi
fi
@@ -1166,7 +1194,9 @@ manager_global() {
" imagerepo: '$IMAGEREPO'"\
" pipeline: 'redis'"\
"sensoroni:"\
- " sensor_checkin_interval_ms: $SENSOR_CHECKIN_INTERVAL_MS"\
+ " node_address: '$MAINIP'"\
+ " node_description: '$NODE_DESCRIPTION'"\
+ " node_checkin_interval_ms: $NODE_CHECKIN_INTERVAL_MS"\
"strelka:"\
" enabled: $STRELKA"\
" rules: 1"\
@@ -1319,7 +1349,7 @@ elasticsearch_pillar() {
parse_install_username() {
# parse out the install username so things copy correctly
- INSTALLUSERNAME=$(pwd | sed -E 's/\// /g' | awk '{ print $2 }')
+ INSTALLUSERNAME=${SUDO_USER:-${USER}}
}
patch_pillar() {
@@ -1741,6 +1771,19 @@ set_network_dev_status_list() {
set_main_ip() {
MAINIP=$(ip route get 1 | awk '{print $7;exit}')
+ MNIC_IP=$(ip a s "$MNIC" | grep -oE 'inet [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' | cut -d' ' -f2)
+}
+
+compare_main_nic_ip() {
+ if [[ "$MAINIP" != "$MNIC_IP" ]]; then
+ read -r -d '' message <<- EOM
+ The IP being routed by Linux is not the IP address assigned to the management interface ($MNIC).
+
+ This is not a supported configuration, please remediate and rerun setup.
+ EOM
+ whiptail --title "Security Onion Setup" --msgbox "$message" 10 75
+ kill -SIGINT "$(ps --pid $$ -oppid=)"; exit 1
+ fi
}
# Add /usr/sbin to everyone's path
@@ -1914,7 +1957,7 @@ set_initial_firewall_policy() {
;;
'HEAVYNODE')
ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/salt/common/tools/sbin/so-firewall includehost sensor "$MAINIP"
- ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/salt/common/tools/sbin/so-firewall --apply includehost search_node "$MAINIP"
+ ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/salt/common/tools/sbin/so-firewall --apply includehost heavy_node "$MAINIP"
ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/pillar/data/addtotab.sh sensorstab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm" "$INTERFACE"
ssh -i /root/.ssh/so.key soremote@"$MSRV" sudo $default_salt_dir/pillar/data/addtotab.sh nodestab "$MINION_ID" "$MAINIP" "$num_cpu_cores" "$random_uid" "$MNIC" "$filesystem_root" "$filesystem_nsm"
;;
diff --git a/setup/so-setup b/setup/so-setup
index 8dcce0e9b..5b751d124 100755
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -54,32 +54,32 @@ while [[ $# -gt 0 ]]; do
esac
done
+if ! [ -f /root/install_opt ] && [ -d /root/manager_setup/securityonion ] && [[ $(pwd) != /root/manager_setup/securityonion/setup ]]; then
+ exec bash /root/manager_setup/securityonion/setup/so-setup "$@"
+fi
+
if [[ -f /root/accept_changes ]]; then
is_reinstall=true
# Move last setup log to backup
mv "$setup_log" "$setup_log.bak"
+ [ -f "$error_log" ] && mv "$error_log" "$error_log.bak"
fi
-# Begin Installation pre-processing
-parse_install_username
-title "Initializing Setup"
-info "Installing as the $INSTALLUSERNAME user"
+if ! [ -f /root/install_opt ]; then
+ # Begin Installation pre-processing
+ parse_install_username
-analyze_system
+ title "Initializing Setup"
+ info "Installing as the $INSTALLUSERNAME user"
+
+ analyze_system
+fi
automated=no
function progress() {
local title='Security Onion Install'
- if grep -q -E "ERROR|Result: False" $setup_log || [[ -s /var/spool/mail/root ]]; then
- if [[ -s /var/spool/mail/root ]]; then
- echo '[ ERROR ] /var/spool/mail/root grew unexpectedly' >> $setup_log 2>&1
- fi
-
- export SO_ERROR=1
- title="Error found, please check $setup_log"
- fi
if [ $automated == no ]; then
whiptail --title "$title" --gauge 'Please wait while installing...' 6 60 0 # append to text
@@ -152,14 +152,18 @@ if [ "$automated" == no ]; then
fi
fi
-if (whiptail_you_sure); then
- true
-else
- echo "User cancelled setup." | tee $setup_log
- whiptail_cancel
-fi
+if ! [ -f /root/install_opt ]; then
+ if (whiptail_you_sure); then
+ true
+ else
+ echo "User cancelled setup." | tee "$setup_log"
+ whiptail_cancel
+ fi
-whiptail_install_type
+ whiptail_install_type
+else
+ install_type=$(cat /root/install_opt)
+fi
if [ "$install_type" = 'EVAL' ]; then
is_node=true
@@ -172,7 +176,6 @@ elif [ "$install_type" = 'STANDALONE' ]; then
is_distmanager=true
is_node=true
is_sensor=true
- is_smooshed=true
elif [ "$install_type" = 'MANAGERSEARCH' ]; then
is_manager=true
is_distmanager=true
@@ -190,7 +193,6 @@ elif [ "$install_type" = 'HEAVYNODE' ]; then
is_node=true
is_minion=true
is_sensor=true
- is_smooshed=true
elif [ "$install_type" = 'FLEET' ]; then
is_minion=true
is_fleet_standalone=true
@@ -200,9 +202,7 @@ elif [ "$install_type" = 'HELIXSENSOR' ]; then
elif [ "$install_type" = 'IMPORT' ]; then
is_import=true
elif [ "$install_type" = 'ANALYST' ]; then
- cd .. || exit 255
- ./so-analyst-install
- exit 0
+ is_analyst=true
fi
# Say yes to the dress if its an ISO install
@@ -211,56 +211,82 @@ if [[ "$setup_type" == 'iso' ]]; then
fi
# Check if this is an airgap install
-
-if [[ $is_manager ]]; then
- if [[ $is_iso ]]; then
- whiptail_airgap
- if [[ "$INTERWEBS" == 'AIRGAP' ]]; then
- is_airgap=true
- fi
- fi
+if [[ $is_manager && $is_iso ]]; then
+ whiptail_airgap
+ if [[ "$INTERWEBS" == 'AIRGAP' ]]; then
+ is_airgap=true
+ fi
fi
-if [[ $is_manager && $is_sensor ]]; then
- check_requirements "standalone"
-elif [[ $is_fleet_standalone ]]; then
- check_requirements "dist" "fleet"
-elif [[ $is_sensor && ! $is_eval ]]; then
- check_requirements "dist" "sensor"
-elif [[ $is_distmanager || $is_minion ]] && [[ ! $is_import ]]; then
- check_requirements "dist"
-elif [[ $is_import ]]; then
- check_requirements "import"
+if ! [ -f /root/install_opt ]; then
+ if [[ $is_manager && $is_sensor ]]; then
+ check_requirements "standalone"
+ elif [[ $is_fleet_standalone ]]; then
+ check_requirements "dist" "fleet"
+ elif [[ $is_sensor && ! $is_eval ]]; then
+ check_requirements "dist" "sensor"
+ elif [[ $is_distmanager || $is_minion ]] && [[ ! $is_import ]]; then
+ check_requirements "dist"
+ elif [[ $is_import ]]; then
+ check_requirements "import"
+ fi
+
+ case "$setup_type" in
+ 'iso')
+ whiptail_set_hostname
+ whiptail_management_nic
+ whiptail_dhcp_or_static
+
+ if [ "$address_type" != 'DHCP' ]; then
+ whiptail_management_interface_ip
+ whiptail_management_interface_mask
+ whiptail_management_interface_gateway
+ whiptail_management_interface_dns
+ whiptail_management_interface_dns_search
+ fi
+ ;;
+ 'network')
+ whiptail_network_notice
+ whiptail_dhcp_warn
+ whiptail_set_hostname
+ whiptail_management_nic
+ ;;
+ esac
+
+ if [[ $is_minion ]]; then
+ whiptail_management_server
+ fi
+
+ if [[ $is_minion || $is_iso ]]; then
+ whiptail_management_interface_setup
+ fi
+
+ if [[ "$setup_type" == 'iso' ]]; then
+ # Init networking so rest of install works
+ set_hostname
+ set_management_interface
+ fi
+
+ if [[ $is_minion ]]; then
+ [ "$automated" == no ] && copy_ssh_key >> $setup_log 2>&1
+ fi
+
+ if [[ $is_minion ]] && ! (compare_versions); then
+ info "Installer version mismatch, downloading correct version from manager"
+ echo "$install_type" > /root/install_opt
+ download_repo_tarball >> "$setup_log" 2>&1
+ exec bash /root/manager_setup/securityonion/setup/so-setup "$@"
+ fi
+
+ if [[ $is_analyst ]]; then
+ cd .. || exit 255
+ exec bash so-analyst-install
+ fi
+
+else
+ rm -rf /root/install_opt >> "$setup_log" 2>&1
fi
-if [[ ! $is_import ]]; then
- whiptail_patch_schedule
-fi
-
-case "$setup_type" in
- 'iso')
- whiptail_set_hostname
- whiptail_management_nic
- whiptail_dhcp_or_static
-
- if [ "$address_type" != 'DHCP' ]; then
- whiptail_management_interface_ip
- whiptail_management_interface_mask
- whiptail_management_interface_gateway
- whiptail_management_interface_dns
- whiptail_management_interface_dns_search
- fi
-
- #collect_adminuser_inputs
- ;;
- 'network')
- whiptail_network_notice
- whiptail_dhcp_warn
- whiptail_set_hostname
- whiptail_management_nic
- ;;
-esac
-
short_name=$(echo "$HOSTNAME" | awk -F. '{print $1}')
MINION_ID=$(echo "${short_name}_${install_type}" | tr '[:upper:]' '[:lower:]')
@@ -327,6 +353,10 @@ if [[ $is_helix || $is_sensor || $is_import ]]; then
calculate_useable_cores
fi
+if [[ ! $is_import ]]; then
+ whiptail_patch_schedule
+fi
+
whiptail_homenet_manager
whiptail_dockernet_check
@@ -372,10 +402,6 @@ if [[ $is_distmanager || ( $is_sensor || $is_node || $is_fleet_standalone ) && !
fi
fi
-if [[ $is_minion ]]; then
- whiptail_management_server
-fi
-
if [[ $is_distmanager ]]; then
collect_soremote_inputs
fi
@@ -432,16 +458,31 @@ echo "1" > /root/accept_changes
trap 'catch $LINENO' SIGUSR1
catch() {
- info "Fatal error occurred at $2 in so-setup, failing setup."
+ info "Fatal error occurred at $1 in so-setup, failing setup."
+ grep --color=never "ERROR" "$setup_log" > "$error_log"
whiptail_setup_failed
exit
}
+# Init networking so rest of install works
+if [[ -n "$TURBO" ]]; then
+ use_turbo_proxy
+fi
+
+disable_ipv6
+
+if [[ "$setup_type" != 'iso' ]]; then
+ set_hostname >> $setup_log 2>&1
+fi
+
+if [[ $is_minion ]]; then
+ add_mngr_ip_to_hosts
+fi
+
# This block sets REDIRECTIT which is used by a function outside the below subshell
-{
- set_main_ip;
- set_redirect;
-} >> $setup_log 2>&1
+ set_main_ip >> $setup_log 2>&1
+ compare_main_nic_ip
+ set_redirect >> $setup_log 2>&1
# Begin install
{
@@ -457,27 +498,8 @@ catch() {
reinstall_init
fi
- if [[ -n "$TURBO" ]]; then
- use_turbo_proxy
- fi
-
- if [[ "$setup_type" == 'iso' ]]; then
- # Init networking so rest of install works
- set_hostname >> $setup_log 2>&1
- set_management_interface
- fi
-
- disable_ipv6
disable_auto_start
- if [[ "$setup_type" != 'iso' ]]; then
- set_hostname >> $setup_log 2>&1
- fi
-
- if [[ $is_minion ]]; then
- add_mngr_ip_to_hosts
- fi
-
{
mark_version;
clear_manager;
@@ -500,7 +522,6 @@ catch() {
if [[ $is_minion || $is_import ]]; then
set_updates >> $setup_log 2>&1
- [ "$automated" == no ] && copy_ssh_key >> $setup_log 2>&1
fi
if [[ $is_manager && $is_airgap ]]; then
@@ -779,21 +800,32 @@ success=$(tail -10 $setup_log | grep Failed | awk '{ print $2}')
if [[ $success != 0 ]]; then SO_ERROR=1; fi
# Check entire setup log for errors or unexpected salt states and ensure cron jobs are not reporting errors to root's mailbox
-if grep -q -E "ERROR|Result: False" $setup_log || [[ -s /var/spool/mail/root && "$setup_type" == "iso" ]]; then SO_ERROR=1; fi
+if grep -q -E "ERROR|Result: False" $setup_log || [[ -s /var/spool/mail/root && "$setup_type" == "iso" ]]; then
+ SO_ERROR=1
+
+ grep --color=never "ERROR" "$setup_log" > "$error_log"
+fi
if [[ -n $SO_ERROR ]]; then
echo "Errors detected during setup; skipping post-setup steps to allow for analysis of failures." >> $setup_log 2>&1
+
SKIP_REBOOT=1
whiptail_setup_failed
+
else
echo "Successfully completed setup! Continuing with post-installation steps" >> $setup_log 2>&1
{
export percentage=95 # set to last percentage used in previous subshell
if [[ -n $ALLOW_ROLE && -n $ALLOW_CIDR ]]; then
- set_progress_str 98 "Running so-allow -${ALLOW_ROLE} for ${ALLOW_CIDR}"
+ set_progress_str 97 "Running so-allow -${ALLOW_ROLE} for ${ALLOW_CIDR}"
IP=$ALLOW_CIDR so-allow -$ALLOW_ROLE >> $setup_log 2>&1
fi
+ if [[ $is_manager ]]; then
+ set_progress_str 98 "Generating archive for setup directory"
+ generate_repo_tarball >> "$setup_log" 2>&1
+ fi
+
if [[ $THEHIVE == 1 ]]; then
set_progress_str 99 'Waiting for TheHive to start up'
check_hive_init >> $setup_log 2>&1
@@ -804,6 +836,6 @@ else
echo "Post-installation steps have completed." >> $setup_log 2>&1
fi
-install_cleanup >> $setup_log 2>&1
+install_cleanup >> "$setup_log" 2>&1
if [[ -z $SKIP_REBOOT ]]; then shutdown -r now; else exit; fi
diff --git a/setup/so-variables b/setup/so-variables
index 83b9b4325..2223fe106 100644
--- a/setup/so-variables
+++ b/setup/so-variables
@@ -21,6 +21,9 @@ export node_es_port
setup_log="/root/sosetup.log"
export setup_log
+error_log="/root/errors.log"
+export error_log
+
filesystem_root=$(df / | awk '$3 ~ /[0-9]+/ { print $2 * 1000 }')
export filesystem_root
diff --git a/setup/so-whiptail b/setup/so-whiptail
index 11d968910..a37340764 100755
--- a/setup/so-whiptail
+++ b/setup/so-whiptail
@@ -35,7 +35,7 @@ whiptail_basic_zeek() {
[ -n "$TESTING" ] && return
- if [[ $is_smooshed ]]; then
+ if [[ $is_node && $is_sensor && ! $is_eval ]]; then
local PROCS=$(expr $lb_procs / 2)
if [ "$PROCS" -lt 1 ]; then PROCS=1; else PROCS=$PROCS; fi
else
@@ -53,7 +53,7 @@ whiptail_basic_suri() {
[ -n "$TESTING" ] && return
- if [[ $is_smooshed ]]; then
+ if [[ $is_node && $is_sensor && ! $is_eval ]]; then
local PROCS=$(expr $lb_procs / 2)
if [ "$PROCS" -lt 1 ]; then PROCS=1; else PROCS=$PROCS; fi
else
@@ -77,7 +77,7 @@ whiptail_zeek_pins() {
cpu_core_list_whiptail+=("$item" "OFF")
done
- if [[ $is_smooshed ]]; then
+ if [[ $is_node && $is_sensor && ! $is_eval ]]; then
local PROCS=$(expr $lb_procs / 2)
if [ "$PROCS" -lt 1 ]; then PROCS=1; else PROCS=$PROCS; fi
else
@@ -345,7 +345,7 @@ whiptail_requirements_error() {
if [[ $(echo "$requirement_needed" | tr '[:upper:]' '[:lower:]') == 'nics' ]]; then
whiptail --title "Security Onion Setup" \
- --msgbox "This machine currently has $current_val $requirement_needed, but needs $needed_val to meet minimum requirements. Press OK to exit setup and reconfigure the machine." 10 75
+ --msgbox "This machine currently has $current_val $requirement_needed, but needs $needed_val to meet minimum requirements. Select OK to exit setup and reconfigure the machine." 10 75
# Same as whiptail_cancel, but changed the wording to exit instead of cancel.
whiptail --title "Security Onion Setup" --msgbox "Exiting Setup. No changes have been made." 8 75
@@ -359,7 +359,7 @@ whiptail_requirements_error() {
exit
else
whiptail --title "Security Onion Setup" \
- --yesno "This machine currently has $current_val $requirement_needed, but needs $needed_val to meet minimum requirements. Press YES to continue anyway, or press NO to cancel." 10 75
+ --yesno "This machine currently has $current_val $requirement_needed, but needs $needed_val to meet minimum requirements. Select YES to continue anyway, or select NO to cancel." 10 75
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
@@ -380,7 +380,7 @@ whiptail_storage_requirements() {
Visit https://docs.securityonion.net/en/2.1/hardware.html for more information.
- Press YES to continue anyway, or press NO to cancel.
+ Select YES to continue anyway, or select NO to cancel.
EOM
whiptail \
@@ -441,7 +441,7 @@ whiptail_dhcp_warn() {
[ -n "$TESTING" ] && return
if [[ $setup_type == "iso" ]]; then
- local interaction_text="Press YES to keep DHCP or NO to go back."
+ local interaction_text="Select YES to keep DHCP or NO to go back."
local window_type="yesno"
else
local interaction_text="Press ENTER to continue."
@@ -765,6 +765,31 @@ whiptail_management_nic() {
}
+whiptail_management_interface_setup() {
+ [ -n "$TESTING" ] && return
+
+ local minion_msg
+ local msg
+
+ if [[ $is_minion ]]; then
+ minion_msg="copy the ssh key for soremote to the manager"
+ else
+ minion_msg=""
+ fi
+
+ if [[ $is_iso ]]; then
+ if [[ $minion_msg != "" ]]; then
+ msg="initialize networking and $minion_msg"
+ else
+ msg="initialize networking"
+ fi
+ fi
+
+ whiptail --title "Security Onion Setup" --yesno "Setup will now $msg. Select YES to continue or NO to cancel." 8 75
+ local exitstatus=$?
+ whiptail_check_exitstatus $exitstatus
+}
+
whiptail_nids() {
[ -n "$TESTING" ] && return
@@ -795,7 +820,7 @@ whiptail_make_changes() {
[ -n "$TESTING" ] && return
- whiptail --title "Security Onion Setup" --yesno "We are going to set this machine up as a $install_type. Please press YES to make changes or NO to cancel." 8 75
+ whiptail --title "Security Onion Setup" --yesno "We are going to set this machine up as a $install_type. Please select YES to make changes or NO to cancel." 8 75
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
@@ -909,7 +934,7 @@ whiptail_network_notice() {
[ -n "$TESTING" ] && return
- whiptail --title "Security Onion Setup" --yesno "Since this is a network install we assume the management interface, DNS, Hostname, etc are already set up. Press YES to continue." 8 75
+ whiptail --title "Security Onion Setup" --yesno "Since this is a network install we assume the management interface, DNS, Hostname, etc are already set up. Select YES to continue." 8 75
local exitstatus=$?
whiptail_check_exitstatus $exitstatus
@@ -1259,7 +1284,20 @@ whiptail_setup_failed() {
[ -n "$TESTING" ] && return
- whiptail --title "Security Onion Setup" --msgbox "Install had a problem. Please see $setup_log for details. Press Ok to exit." 8 75
+ local check_err_msg
+ local height
+
+ [ -f "$error_log" ] && check_err_msg="A summary of errors can be found in $error_log.\n"
+
+ if [[ -n $check_err_msg ]]; then height=11; else height=10; fi
+
+ read -r -d '' message <<- EOM
+ Install had a problem. Please see $setup_log for details.\n
+ $check_err_msg
+ Press Ok to exit.
+ EOM
+
+ whiptail --title "Security Onion Setup" --msgbox "$message" $height 75
}
whiptail_shard_count() {
@@ -1325,11 +1363,11 @@ whiptail_suricata_pins() {
readarray -t filtered_core_list <<< "$(echo "${cpu_core_list[@]}" "${ZEEKPINS[@]}" | xargs -n1 | sort | uniq -u | awk '{print $1}')"
local filtered_core_str=()
- for item in "${filtered_core_list[@]}"; do
- filtered_core_str+=("$item" "")
- done
+ for item in "${filtered_core_list[@]}"; do
+ filtered_core_str+=("$item" "")
+ done
- if [[ $is_smooshed ]]; then
+ if [[ $is_node && $is_sensor && ! $is_eval ]]; then
local PROCS=$(expr $lb_procs / 2)
if [ "$PROCS" -lt 1 ]; then PROCS=1; else PROCS=$PROCS; fi
else