From 19cd57bce0f2c72fd1aeab9e13503ce40cb77a27 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 22 May 2020 14:44:57 -0400 Subject: [PATCH 01/20] Support automated setup --- setup/automation/pm_standalone_defaults | 75 +++++++++++++++++++++++++ setup/so-functions | 3 + setup/so-setup | 6 ++ 3 files changed, 84 insertions(+) create mode 100644 setup/automation/pm_standalone_defaults diff --git a/setup/automation/pm_standalone_defaults b/setup/automation/pm_standalone_defaults new file mode 100644 index 000000000..b5a6258ff --- /dev/null +++ b/setup/automation/pm_standalone_defaults @@ -0,0 +1,75 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +TESTING=true + +address_type=DHCP +ADMINUSER=onionuser +ADMINPASS1=onionuser +ADMINPASS2=onionuser +BASICBRO=7 +BASICSURI=7 +# BLOGS= +BNICS=eth1 +BROVERSION=ZEEK +# CURCLOSEDAYS= +# EVALADVANCED=BASIC +GRAFANA=1 +# HELIXAPIKEY= +HNMASTER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 +HNSENSOR=inherit +HOSTNAME=standalone +install_type=STANDALONE +# LSINPUTBATCHCOUNT= +# LSINPUTTHREADS= +# LSPIPELINEBATCH= +# LSPIPELINEWORKERS= +MASTERADV=BASIC +MASTERUPDATES=1 +# MDNS= +# MGATEWAY= +# MIP= +# MMASK= +MNIC=eth0 +# MSEARCH= +# MSRV= +# MTU= +NAVIGATOR=1 +NIDS=Suricata +# NODE_ES_HEAP_SIZE= +# NODE_LS_HEAP_SIZE= +NODESETUP=NODEBASIC +NSMSETUP=BASIC +NODEUPDATES=MASTER +# OINKCODE= +OSQUERY=1 +# PATCHSCHEDULEDAYS= +# PATCHSCHEDULEHOURS= +PATCHSCHEDULENAME=auto +PLAYBOOK=1 +# REDIRECTHOST= +REDIRECTINFO=IP +RULESETUP=ETOPEN +# SHARDCOUNT= +SOREMOTEPASS1=onionuser +SOREMOTEPASS2=onionuser +STRELKA=1 +THEHIVE=1 +WAZUH=1 +WEBUSER=onionuser@somewhere.invalid +WEBPASSWD1=onionuser +WEBPASSWD2=onionuser \ No newline at end of file diff --git a/setup/so-functions b/setup/so-functions index a20953035..185f41d89 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -550,6 +550,9 @@ detect_os() { disable_onion_user() { # Disable the default account cause security. usermod -L onion + + # Remove the automated setup script from crontab, if it exists + crontab -u onion -r } disable_misc_network_features() { diff --git a/setup/so-setup b/setup/so-setup index 406d69763..3e54d3f66 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -24,6 +24,12 @@ source ./so-variables setup_type=$1 export setup_type +automation=$2 +if [[ -f automation/$automation && $(basename $automation) == $automation ]]; then + echo "Preselecting variable values based on automated setup: $automation" + source automation/$automation +fi + case "$setup_type" in iso | network) # Accepted values echo "Beginning Security Onion $setup_type install" From 13423bed18eca71c101252a0dc7d4683051eb5fb Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 22 May 2020 16:35:16 -0400 Subject: [PATCH 02/20] [fix] Add indent in proxy.conf --- setup/so-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 04b2652a5..ddf265153 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1492,8 +1492,8 @@ use_proxy() { else printf '%s\n'\ "Acquire {"\ - "HTTP::proxy \"${proxy_addr}:3142\";"\ - "HTTPS::proxy \"${proxy_addr}:3142\";"\ + " HTTP::proxy \"${proxy_addr}:3142\";"\ + " HTTPS::proxy \"${proxy_addr}:3142\";"\ "}" > /etc/apt/apt.conf.d/proxy.conf fi } From 56f5fbdf6b095445094de6a379b80fb2d690b085 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 22 May 2020 17:11:08 -0400 Subject: [PATCH 03/20] Ingest pipeline commid fix for conn logs --- salt/elasticsearch/files/ingest/common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/common b/salt/elasticsearch/files/ingest/common index e70d5e2d8..a65742f99 100644 --- a/salt/elasticsearch/files/ingest/common +++ b/salt/elasticsearch/files/ingest/common @@ -38,7 +38,7 @@ { "rename": { "field": "module", "target_field": "event.module", "ignore_missing": true } }, { "rename": { "field": "dataset", "target_field": "event.dataset", "ignore_missing": true } }, { "rename": { "field": "category", "target_field": "event.category", "ignore_missing": true } }, - { "rename": { "field": "message2.community_id", "target_field": "network.community_id", "ignore_missing": true } }, + { "rename": { "field": "message2.community_id", "target_field": "network.community_id", "ignore_failure": true, "ignore_missing": true } }, { "remove": { "field": [ "index_name_prefix", "message2", "type" ], From 1c207afb31bd321e25a1e7e72586b7df438c3870 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Sat, 23 May 2020 12:00:24 -0400 Subject: [PATCH 04/20] Define packages in the common state --- salt/common/init.sls | 85 ++++++++++++++++++++++++++++++++++++++++---- setup/so-functions | 12 ++++--- 2 files changed, 85 insertions(+), 12 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 82ac4a062..a8c791c7f 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -28,21 +28,92 @@ salttmp: - group: 939 - makedirs: True -# Install packages needed for the sensor -sensorpkgs: +# Install epel +{% if grains['os'] == 'CentOS' %} +epel: pkg.installed: - - skip_suggestions: False + - skip_suggestions: True + - pkgs: + - epel-release +{% endif %} + +# Install common packages +commonpkgs: + pkg.installed: + - skip_suggestions: True - pkgs: - - wget - - jq {% if grains['os'] != 'CentOS' %} - apache2-utils + - wget + - jq + - python3-docker + - docker-ce + - curl + - ca-certificates + - software-properties-common + - apt-transport-https + - openssl + - netcat + - python3-mysqldb + - sqlite3 + - argon2 + - libssl-dev + - python3-dateutil + - python3-m2crypto + - python3-mysqldb + - salt-minion: 2019.2.5+ds-1 + - hold: True + - update_holds: True + {% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' or grains['role'] == 'so-standalone' %} + - salt-master: 2019.2.5+ds-1 + - hold: True + - update_holds: True + - containerd.io: 1.2.13-2 + - hold: True + - update_holds: True + - docker-ce: 5:19.03.9~3-0~ubuntu-bionic + - hold: True + - update_holds: True + {% endif %} + - containerd.io + - docker-ce {% else %} - - net-tools + - wget + - bind-utils + - jq - tcpdump - httpd-tools + - net-tools + - curl + - sqlite + - argon2 + - maridb-devel + - nmap-ncat + - python3 + - python36-docker + - python36-dateutil + - python36-m2crypto + - python36-mysql + - yum-utils + - device-mapper-persistent-data + - lvm2 + - openssl + - salt-minion: 2019.2.5 + - hold: True + - update_holds: True + {% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' or grains['role'] == 'so-standalone' %} + - salt-master: 2019.2.5 + - hold: True + - update_holds: True {% endif %} - + - containerd.io: 1.2.6-3 + - hold: True + - update_holds: True + - docker-ce: 19.03.9-3 + - hold: True + - update_holds: True + {% endif %}%} + # Always keep these packages up to date alwaysupdated: diff --git a/setup/so-functions b/setup/so-functions index ddf265153..702c3cebd 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -19,7 +19,7 @@ source ./so-whiptail source ./so-variables source ./so-common-functions -SOVERSION=1.3.0 +SOVERSION=1.4.0 accept_salt_key_remote() { systemctl restart salt-minion @@ -514,7 +514,7 @@ detect_os() { # Install bind-utils so the host command exists if ! command -v host > /dev/null 2>&1; then echo "Installing required packages to run installer" - yum -y install bind-utils >> "$setup_log" 2>&1 + yum -y install bind-utils yum-plugin-versionlock >> "$setup_log" 2>&1 fi @@ -583,7 +583,9 @@ docker_install() { { yum clean expire-cache; yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo; - yum -y install docker-ce; + yum -y install docker-ce-19.03.9-3 containerd.io-1.2.6-3; + yum versionlock docker-ce; + yum versionlock containerd.io } >> "$setup_log" 2>&1 else @@ -687,7 +689,7 @@ docker_seed_registry() { # Tag it with the new registry destination docker tag soshybridhunter/"$i" "$HOSTNAME":5000/soshybridhunter/"$i" docker push "$HOSTNAME":5000/soshybridhunter/"$i" - docker rmi soshybridhunter/"$i" + #docker rmi soshybridhunter/"$i" } >> "$setup_log" 2>&1 done else @@ -1107,7 +1109,7 @@ saltify() { yum -y update exclude=salt*; systemctl enable salt-minion; } >> "$setup_log" 2>&1 - echo "exclude=salt*" >> /etc/yum.conf + yum versionlock salt* else DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade >> "$setup_log" 2>&1 From 31b3563fb3608bc9b104bd6456bca84bbe8e5f81 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Sun, 24 May 2020 10:56:30 -0400 Subject: [PATCH 05/20] Fix package versions --- salt/common/init.sls | 8 ++++---- setup/so-functions | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index a8c791c7f..17cbb1608 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -98,18 +98,18 @@ commonpkgs: - device-mapper-persistent-data - lvm2 - openssl - - salt-minion: 2019.2.5 + - salt-minion: 2019.2.5.el7 - hold: True - update_holds: True {% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' or grains['role'] == 'so-standalone' %} - - salt-master: 2019.2.5 + - salt-master: 2019.2.5.el7 - hold: True - update_holds: True {% endif %} - - containerd.io: 1.2.6-3 + - containerd.io: 1.2.6-3.el7 - hold: True - update_holds: True - - docker-ce: 19.03.9-3 + - docker-ce: 19.03.9-3.el7 - hold: True - update_holds: True {% endif %}%} diff --git a/setup/so-functions b/setup/so-functions index 702c3cebd..bf6db26be 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -583,9 +583,9 @@ docker_install() { { yum clean expire-cache; yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo; - yum -y install docker-ce-19.03.9-3 containerd.io-1.2.6-3; - yum versionlock docker-ce; - yum versionlock containerd.io + yum -y install docker-ce-19.03.9-3.el7 containerd.io-1.2.6-3.el7; + yum versionlock docker-ce-19.03.9-3.el7; + yum versionlock containerd.io-1.2.6-3.el7 } >> "$setup_log" 2>&1 else From ad97092589f8d46600a4ce695d83155895cac84d Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sun, 24 May 2020 21:38:37 -0400 Subject: [PATCH 06/20] When automating setup installation wait for network availability --- setup/so-setup | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-setup b/setup/so-setup index b3c141fae..9384c5e96 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -28,6 +28,7 @@ automation=$2 if [[ -f automation/$automation && $(basename $automation) == $automation ]]; then echo "Preselecting variable values based on automated setup: $automation" source automation/$automation + sleep 30 # Re-implement with network availability probe fi case "$setup_type" in From e09027e7954016e43a3f0b01ac7109e43fccaeab Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 26 May 2020 13:00:03 +0000 Subject: [PATCH 07/20] update Strelka cron --- salt/strelka/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index 8bdbd8274..a9842924d 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -112,5 +112,5 @@ strelka_filestream: strelka_zeek_extracted_sync: cron.present: - user: root - - name: mv /nsm/zeek/extracted/complete/* /nsm/strelka + - name: [ -d /nsm/zeek/extracted/complete/ ] && mv /nsm/zeek/extracted/complete/* /nsm/strelka/ > /dev/null 2>&1 - minute: '*' From 330de4624955d2fbe0bce04869fb55f3e6376dbe Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 26 May 2020 13:19:15 +0000 Subject: [PATCH 08/20] update SOCtopus config for hostname/ip --- salt/soctopus/files/SOCtopus.conf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/soctopus/files/SOCtopus.conf b/salt/soctopus/files/SOCtopus.conf index f2415d010..e5878cb70 100644 --- a/salt/soctopus/files/SOCtopus.conf +++ b/salt/soctopus/files/SOCtopus.conf @@ -1,9 +1,9 @@ -{%- set ip = salt['pillar.get']('static:masterip', '') %} +{%- set MASTER = salt['pillar.get']('master:url_base', '') %} {%- set HIVEKEY = salt['pillar.get']('static:hivekey', '') %} {%- set CORTEXKEY = salt['pillar.get']('static:cortexorguserkey', '') %} [es] -es_url = http://{{ip}}:9200 +es_url = http://{{MASTER}}:9200 es_user = YOURESUSER es_pass = YOURESPASS es_index_pattern = so-* @@ -11,7 +11,7 @@ es_verifycert = no [cortex] auto_analyze_alerts = no -cortex_url = https://{{ip}}/cortex/ +cortex_url = https://{{MASTER}}/cortex/ cortex_key = {{ CORTEXKEY }} supported_analyzers = Urlscan_io_Search,CERTatPassiveDNS @@ -32,7 +32,7 @@ grr_user = YOURGRRUSER grr_pass = YOURGRRPASS [hive] -hive_url = https://{{ip}}/thehive/ +hive_url = https://{{MASTER}}/thehive/ hive_key = {{ HIVEKEY }} hive_tlp = 3 hive_verifycert = no @@ -59,7 +59,7 @@ slack_url = YOURSLACKWORKSPACE slack_webhook = YOURSLACKWEBHOOK [playbook] -playbook_url = https://{{ip}}/playbook +playbook_url = https://{{MASTER}}/playbook playbook_key = de6639318502476f2fa5aa06f43f51fb389a3d7f playbook_verifycert = no From c28936d9e42ea3a23f5f17c49b0379981cbb6705 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 26 May 2020 09:40:29 -0400 Subject: [PATCH 09/20] temporarily abort automated installation --- setup/so-setup | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-setup b/setup/so-setup index 9384c5e96..69c5763f9 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -27,6 +27,7 @@ export setup_type automation=$2 if [[ -f automation/$automation && $(basename $automation) == $automation ]]; then echo "Preselecting variable values based on automated setup: $automation" + exit 1 source automation/$automation sleep 30 # Re-implement with network availability probe fi From 3eb62287ac6c2195cf2536bc4909a150ca232b6e Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 26 May 2020 14:00:16 +0000 Subject: [PATCH 10/20] update Curator config for index transition --- salt/curator/files/action/close.yml | 5 ++--- salt/curator/files/action/delete.yml | 4 ++-- salt/curator/files/bin/so-curator-closed-delete-delete | 8 ++++---- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/salt/curator/files/action/close.yml b/salt/curator/files/action/close.yml index dfe5519e8..a65e9af3d 100644 --- a/salt/curator/files/action/close.yml +++ b/salt/curator/files/action/close.yml @@ -24,9 +24,8 @@ actions: disable_action: False filters: - filtertype: pattern - kind: prefix - value: logstash- - exclude: + kind: regex + value: '^(logstash-.*|so-.*)$' - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/delete.yml b/salt/curator/files/action/delete.yml index e6f2f3833..030bbbfac 100644 --- a/salt/curator/files/action/delete.yml +++ b/salt/curator/files/action/delete.yml @@ -20,8 +20,8 @@ actions: disable_action: False filters: - filtertype: pattern - kind: prefix - value: logstash- + kind: regex + value: '^(logstash-.*|so-.*)$' - filtertype: space source: creation_date use_age: True diff --git a/salt/curator/files/bin/so-curator-closed-delete-delete b/salt/curator/files/bin/so-curator-closed-delete-delete index b0ec62424..3d397defc 100755 --- a/salt/curator/files/bin/so-curator-closed-delete-delete +++ b/salt/curator/files/bin/so-curator-closed-delete-delete @@ -33,17 +33,17 @@ LOG="/opt/so/log/curator/so-curator-closed-delete.log" # Check for 2 conditions: # 1. Are Elasticsearch indices using more disk space than LOG_SIZE_LIMIT? -# 2. Are there any closed logstash- indices that we can delete? +# 2. Are there any closed logstash-, or so- indices that we can delete? # If both conditions are true, keep on looping until one of the conditions is false. while [[ $(du -hs --block-size=1GB /nsm/elasticsearch/nodes | awk '{print $1}' ) -gt "{{LOG_SIZE_LIMIT}}" ]] && -curl -s {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices | grep "^ close logstash-" > /dev/null; do +curl -s {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices | grep -E "^ close (logstash-|so-)" > /dev/null; do # We need to determine OLDEST_INDEX. - # First, get the list of closed indices that are prefixed with "logstash-". + # First, get the list of closed indices that are prefixed with "logstash-" or "so-". # For example: logstash-ids-YYYY.MM.DD # Then, sort by date by telling sort to use hyphen as delimiter and then sort on the third field. # Finally, select the first entry in that sorted list. - OLDEST_INDEX=$(curl -s {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices | grep "^ close logstash-" | awk '{print $2}' | sort -t- -k3 | head -1) + OLDEST_INDEX=$(curl -s {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices | grep -E "^ close (logstash-|so-)" | awk '{print $2}' | sort -t- -k3 | head -1) # Now that we've determined OLDEST_INDEX, ask Elasticsearch to delete it. curl -XDELETE {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/${OLDEST_INDEX} From 97306d3acdbf41e311c52f9afb5fdf1e9d39c4a6 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 26 May 2020 14:05:33 +0000 Subject: [PATCH 11/20] rename indices --- salt/common/tools/sbin/so-elastalert-create | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-elastalert-create b/salt/common/tools/sbin/so-elastalert-create index fbe9527a7..0270503bf 100755 --- a/salt/common/tools/sbin/so-elastalert-create +++ b/salt/common/tools/sbin/so-elastalert-create @@ -166,8 +166,7 @@ cat << EOF What elasticsearch index do you want to use? Below are the default Index Patterns used in Security Onion: -*:logstash-* -*:logstash-beats-* +*:so-ids-* *:elastalert_status* EOF From 0e51ab41cf83df46eeeac5ee3a1cebd6f4022251 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 26 May 2020 14:18:58 +0000 Subject: [PATCH 12/20] Update ES watermark settings --- salt/elasticsearch/files/elasticsearch.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index 73f3c9239..271ef40cf 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -22,3 +22,7 @@ transport.bind_host: 0.0.0.0 transport.publish_host: {{ nodeip }} transport.publish_port: 9300 {%- endif %} +cluster.routing.allocation.disk.threshold_enabled: true +cluster.routing.allocation.disk.watermark.low: 95% +cluster.routing.allocation.disk.watermark.high: 98% +cluster.routing.allocation.disk.watermark.flood_stage: 98% From 87407c9acefc7766f1f12ec1772822d49f965816 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 26 May 2020 11:21:59 -0400 Subject: [PATCH 13/20] rework common init.sls --- salt/common/init.sls | 72 +++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 41 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 17cbb1608..77801dd1a 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -38,11 +38,11 @@ epel: {% endif %} # Install common packages +{% if grains['os'] != 'CentOS' %} commonpkgs: pkg.installed: - skip_suggestions: True - pkgs: - {% if grains['os'] != 'CentOS' %} - apache2-utils - wget - jq @@ -61,23 +61,19 @@ commonpkgs: - python3-dateutil - python3-m2crypto - python3-mysqldb - - salt-minion: 2019.2.5+ds-1 - - hold: True - - update_holds: True - {% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' or grains['role'] == 'so-standalone' %} - - salt-master: 2019.2.5+ds-1 - - hold: True - - update_holds: True +heldpackages: + pkg.installed: + - pkgs: - containerd.io: 1.2.13-2 - - hold: True - - update_holds: True - docker-ce: 5:19.03.9~3-0~ubuntu-bionic - - hold: True - - update_holds: True - {% endif %} - - containerd.io - - docker-ce - {% else %} + - hold: True + - update_holds: True + +{% else %} +commonpkgs: + pkg.installed: + - skip_suggestions: True + - pkgs: - wget - bind-utils - jq @@ -87,33 +83,27 @@ commonpkgs: - curl - sqlite - argon2 - - maridb-devel + - mariadb-devel - nmap-ncat - python3 - - python36-docker - - python36-dateutil - - python36-m2crypto - - python36-mysql - - yum-utils - - device-mapper-persistent-data - - lvm2 - - openssl - - salt-minion: 2019.2.5.el7 - - hold: True - - update_holds: True - {% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' or grains['role'] == 'so-standalone' %} - - salt-master: 2019.2.5.el7 - - hold: True - - update_holds: True - {% endif %} - - containerd.io: 1.2.6-3.el7 - - hold: True - - update_holds: True - - docker-ce: 19.03.9-3.el7 - - hold: True - - update_holds: True - {% endif %}%} - + - python36-docker + - python36-dateutil + - python36-m2crypto + - python36-mysql + - yum-utils + - device-mapper-persistent-data + - lvm2 + - openssl + +heldpackages: + pkg.installed: + - pkgs: + - containerd.io: 1.2.13-3.2.el7 + - docker-ce: 3:19.03.9-3.el7 + - hold: True + - update_holds: True +{% endif %} + # Always keep these packages up to date alwaysupdated: From 2cb7464086c8208efa7184d935af78cd586c8686 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 26 May 2020 12:01:58 -0400 Subject: [PATCH 14/20] Add TheHive communityid link --- salt/elastalert/files/rules/so/nids2hive.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elastalert/files/rules/so/nids2hive.yaml b/salt/elastalert/files/rules/so/nids2hive.yaml index 68a173fcd..097511d56 100644 --- a/salt/elastalert/files/rules/so/nids2hive.yaml +++ b/salt/elastalert/files/rules/so/nids2hive.yaml @@ -40,7 +40,7 @@ hive_alert_config: title: '{match[rule][name]}' type: 'NIDS' source: 'SecurityOnion' - description: "`Hunting Pivot:` \n\n \n\n `Kibana Dashboard:` \n\n \n\n `IPs: `{match[source][ip]}:{match[source][port]} --> {match[destination][ip]}:{match[destination][port]} \n\n `Signature:`{match[rule][rule]}" + description: "`Hunting Pivot:` \n\n \n\n `Kibana Dashboard - Signature Drilldown:` \n\n \n\n `Kibana Dashboard - Community_ID:` \n\n \n\n `IPs: `{match[source][ip]}:{match[source][port]} --> {match[destination][ip]}:{match[destination][port]} \n\n `Signature:`{match[rule][rule]}" severity: 2 tags: ['{match[rule][uuid]}','{match[source][ip]}','{match[destination][ip]}'] tlp: 3 From 7a657d122989cdc4545ea01c3e1c49fd10a64f18 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Tue, 26 May 2020 12:58:01 -0400 Subject: [PATCH 15/20] add Community ID to default fields in bottom data table in Hunt --- salt/soc/files/soc/soc.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index 6b76e622c..76770e2bd 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -32,7 +32,7 @@ "dateRangeMinutes": 1440, "mostRecentlyUsedLimit": 5, "eventFields": { - "default": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "log.id.uid" ], + "default": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "log.id.uid", "network.community_id" ], "bro_conn": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "protocol", "service", "log.id.uid" ], "bro_dce_rpc": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "endpoint", "named_pipe", "operation", "log.id.uid" ], "bro_dhcp": ["soc_timestamp", "source.ip", "destination.ip", "domain_name", "hostname", "message_types", "log.id.uid" ], From 8723f8785ec2043f06945b3fd0b1d6ca424e5f11 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 26 May 2020 13:05:56 -0400 Subject: [PATCH 16/20] osquery pipeline fix and fail state if errors --- salt/elasticsearch/files/ingest/osquery.query_result | 2 +- salt/elasticsearch/files/so-elasticsearch-pipelines | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/files/ingest/osquery.query_result b/salt/elasticsearch/files/ingest/osquery.query_result index e9cdbe2d3..5b37655f9 100644 --- a/salt/elasticsearch/files/ingest/osquery.query_result +++ b/salt/elasticsearch/files/ingest/osquery.query_result @@ -31,7 +31,7 @@ { "rename": { "field": "message3.columns.remote_port", "target_field": "remote.port", "ignore_missing": true } }, { "rename": { "field": "message3.columns.process_name", "target_field": "process.name", "ignore_missing": true } }, { "rename": { "field": "message3.columns.eventid", "target_field": "event.code", "ignore_missing": true } }, - { "set": { "if": "ctx.message3.columns.?data != null", "field": "dataset", "value": "wel-{{message3.columns.source}}", "override": true } }, + { "set": { "if": "ctx.message3.columns?.data != null", "field": "dataset", "value": "wel-{{message3.columns.source}}", "override": true } }, { "rename": { "field": "message3.columns.winlog.EventData.SubjectUserName", "target_field": "user.name", "ignore_missing": true } }, { "rename": { "field": "message3.columns.winlog.EventData.destinationHostname", "target_field": "destination.hostname", "ignore_missing": true } }, { "rename": { "field": "message3.columns.winlog.EventData.destinationIp", "target_field": "destination.ip", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/so-elasticsearch-pipelines b/salt/elasticsearch/files/so-elasticsearch-pipelines index b1b6db158..514054359 100755 --- a/salt/elasticsearch/files/so-elasticsearch-pipelines +++ b/salt/elasticsearch/files/so-elasticsearch-pipelines @@ -15,6 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +RETURN_CODE=0 ELASTICSEARCH_HOST=$1 ELASTICSEARCH_PORT=9200 @@ -46,7 +47,9 @@ fi cd ${ELASTICSEARCH_INGEST_PIPELINES} echo "Loading pipelines..." -for i in *; do echo $i; curl ${ELASTICSEARCH_AUTH} -XPUT http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done +for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -XPUT http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done echo cd - >/dev/null + +exit $RETURN_CODE \ No newline at end of file From d260224dfea4c7b8c384911a452d641d47a432a8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 26 May 2020 13:39:36 -0400 Subject: [PATCH 17/20] Update reboot language --- setup/so-whiptail | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 559f791b7..72455fc9e 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -958,7 +958,7 @@ whiptail_setup_complete() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --msgbox "Finished $install_type install. Press ENTER to reboot." 8 75 + whiptail --title "Security Onion Setup" --msgbox "Finished $install_type install. Press Ok to reboot." 8 75 install_cleanup >> $setup_log 2>&1 } @@ -967,7 +967,7 @@ whiptail_setup_failed() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --msgbox "Install had a problem. Please see $setup_log for details. Press ENTER to reboot." 8 75 + whiptail --title "Security Onion Setup" --msgbox "Install had a problem. Please see $setup_log for details. Press Ok to reboot." 8 75 install_cleanup >> $setup_log 2>&1 } From 41935996d4384849fd4ffe93924d2c2fb6bd9ff7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 26 May 2020 13:48:36 -0400 Subject: [PATCH 18/20] add ntpdate/ntp --- salt/common/init.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/init.sls b/salt/common/init.sls index 77801dd1a..08ae7fa72 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -45,6 +45,7 @@ commonpkgs: - pkgs: - apache2-utils - wget + - ntp - jq - python3-docker - docker-ce @@ -75,6 +76,7 @@ commonpkgs: - skip_suggestions: True - pkgs: - wget + - ntpdate - bind-utils - jq - tcpdump From 21cd66d109c458b8aa335d3e93143fb9555a8d94 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 26 May 2020 14:05:47 -0400 Subject: [PATCH 19/20] add ntpdate/ntp --- salt/common/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 08ae7fa72..09d71114b 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -45,7 +45,7 @@ commonpkgs: - pkgs: - apache2-utils - wget - - ntp + - ntpdate - jq - python3-docker - docker-ce From 8e1bd32f4d5e9ecdd1b3bb8f3bdf334f1ad08a29 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 26 May 2020 16:11:31 -0400 Subject: [PATCH 20/20] Improve automated installs and remove sleep during progress updates --- setup/so-functions | 4 +--- setup/so-setup | 28 +++++++++++++++++++++++++--- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index bf6db26be..f8d41cfc7 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -968,7 +968,7 @@ parse_options() { fi ;; --proxy=*) - echo "Unimplimented" + echo "Unimplemented" return if [[ $2 != --proxy-user=* ]] || [[ $3 != --proxy-pass=* ]]; then @@ -1268,8 +1268,6 @@ set_progress_str() { '----'\ "$percentage% - ${progress_bar_text^^}"\ "----" >> "$setup_log" 2>&1 - - sleep 5 } sensor_pillar() { diff --git a/setup/so-setup b/setup/so-setup index 69c5763f9..03889606d 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -25,11 +25,33 @@ setup_type=$1 export setup_type automation=$2 + +automated=no + +function progress() { + if [ $automated == no ]; then + whiptail --title "Security Onion Install" --gauge 'Please wait while installing' 6 60 0 + fi +} + if [[ -f automation/$automation && $(basename $automation) == $automation ]]; then echo "Preselecting variable values based on automated setup: $automation" - exit 1 source automation/$automation - sleep 30 # Re-implement with network availability probe + automated=yes + + attempt=1 + attempts=60 + ip a | grep "$MNIC:" | grep "state UP" + while [ $? -ne 0 ]; do + if [ $attempt -gt $attempts ]; then + echo "Network unavailable - setup cannot continue" + exit 1 + fi + echo "Waiting for network to come up (attempt $attempt of $attempts)" + attempt=$((attempt + 1)) + sleep 10; + ip a | grep "$MNIC:" | grep "state UP" + done fi case "$setup_type" in @@ -498,7 +520,7 @@ fi set_progress_str 95 'Verifying setup' salt-call -l info state.highstate >> $setup_log 2>&1 -} | whiptail --title "Hybrid Hunter Install" --gauge "Please wait while installing" 6 60 0 +} | progress success=$(tail -10 $setup_log | grep Failed | awk '{ print $2}') if [[ "$success" = 0 ]]; then