From 39c3e5793fd88d60cf5a2ee670be0e2f4422aa35 Mon Sep 17 00:00:00 2001 From: Dustin Lee Date: Mon, 3 Dec 2018 19:54:33 -0500 Subject: [PATCH 001/110] Added grafana.ini in RO to prep for proxy --- salt/common/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/init.sls b/salt/common/init.sls index f0b4fd8fa..ca771afdf 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -372,6 +372,7 @@ so-grafana: - user: socore - binds: - /nsm/grafana:/var/lib/grafana:rw + - /opt/so/conf/grafana/etc/grafana.ini:/etc/grafana/grafana.ini:ro - /opt/so/conf/grafana/etc/datasources:/etc/grafana/provisioning/datasources:rw - /opt/so/conf/grafana/etc/dashboards:/etc/grafana/provisioning/dashboards:rw - /opt/so/conf/grafana/grafana_dashboards:/etc/grafana/grafana_dashboards:rw From 27dce49a31e238b66e57706d1d5f8108b486e084 Mon Sep 17 00:00:00 2001 From: Dustin Lee Date: Mon, 3 Dec 2018 19:56:09 -0500 Subject: [PATCH 002/110] Uncommented `root_url` and added proxy info --- salt/common/grafana/etc/grafana.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/grafana/etc/grafana.ini b/salt/common/grafana/etc/grafana.ini index adbbeb316..0327b8bc5 100644 --- a/salt/common/grafana/etc/grafana.ini +++ b/salt/common/grafana/etc/grafana.ini @@ -46,7 +46,7 @@ # The full public facing url you use in browser, used for redirects and emails # If you use reverse proxy and sub path specify full url (with sub path) -;root_url = http://localhost:3000 +root_url = %(protocol)s://%(domain)s/grafana/ # Log web requests ;router_logging = false From 935c4470e8c2e5f5cba912b429b2987da3526e35 Mon Sep 17 00:00:00 2001 From: Dustin Lee Date: Mon, 3 Dec 2018 19:59:43 -0500 Subject: [PATCH 003/110] Added /grafana/ and /kibana/ proxy info --- salt/common/nginx/nginx.conf.so-eval | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/salt/common/nginx/nginx.conf.so-eval b/salt/common/nginx/nginx.conf.so-eval index bc8dbf39b..07813d1a1 100644 --- a/salt/common/nginx/nginx.conf.so-eval +++ b/salt/common/nginx/nginx.conf.so-eval @@ -87,7 +87,20 @@ http { # try_files $uri $uri.html /index.html; # } - location / { + location /grafana/ { + rewrite /grafana/(.*) /$1 break; + proxy_pass http://{{ masterip }}:3000/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } + + location /kibana/ { + rewrite /kibana/(.*) /$1 break; proxy_pass http://{{ masterip }}:5601/; proxy_read_timeout 90; proxy_connect_timeout 90; From 40a87d339f62d27349cfc0e8179a855b1f56c27f Mon Sep 17 00:00:00 2001 From: Dustin Lee Date: Mon, 3 Dec 2018 20:00:36 -0500 Subject: [PATCH 004/110] Added /grafana/ and /kibana/ proxy info --- salt/common/nginx/nginx.conf.so-master | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/salt/common/nginx/nginx.conf.so-master b/salt/common/nginx/nginx.conf.so-master index bc8dbf39b..07813d1a1 100644 --- a/salt/common/nginx/nginx.conf.so-master +++ b/salt/common/nginx/nginx.conf.so-master @@ -87,7 +87,20 @@ http { # try_files $uri $uri.html /index.html; # } - location / { + location /grafana/ { + rewrite /grafana/(.*) /$1 break; + proxy_pass http://{{ masterip }}:3000/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } + + location /kibana/ { + rewrite /kibana/(.*) /$1 break; proxy_pass http://{{ masterip }}:5601/; proxy_read_timeout 90; proxy_connect_timeout 90; From 9deaba8be9e34c5e5ba6b03e73d5de361659fd1a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 3 Dec 2018 20:20:57 -0500 Subject: [PATCH 005/110] Mysql Module - Change to new docker --- salt/mysql/init.sls | 2 +- so-setup-network.sh | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/mysql/init.sls b/salt/mysql/init.sls index d5a48eff1..f4b9ad03f 100644 --- a/salt/mysql/init.sls +++ b/salt/mysql/init.sls @@ -43,7 +43,7 @@ mysqldatadir: so-mysql: docker_container.running: - - image: mysql/mysql-server:5.7 + - image: soshybridhunter/so-mysql:HH1.0.5 - hostname: so-mysql - user: socore - port_bindings: diff --git a/so-setup-network.sh b/so-setup-network.sh index aef893040..b44af1c57 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -170,6 +170,10 @@ configure_minion() { if [ $TYPE == 'master' ] || [ $TYPE == 'eval' ]; then echo "master: $HOSTNAME" > /etc/salt/minion echo "id: $HOSTNAME" >> /etc/salt/minion + echo "mysql.host: '$MAINIP'" >> /etc/salt/minion + echo "mysql.port: 3306" >> /etc/salt/minion + echo "mysql.user: 'root'" >> /etc/salt/minion + echo "mysql.pass: '$MYSQLPASS'" >> /etc/salt/minion else echo "master: $MSRV" > /etc/salt/minion echo "id: $HOSTNAME" >> /etc/salt/minion From 8667cad374e907040bca39c9f156932b676e6d1d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 3 Dec 2018 20:26:11 -0500 Subject: [PATCH 006/110] Mysql Module - Fix config location --- salt/mysql/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/mysql/init.sls b/salt/mysql/init.sls index f4b9ad03f..1683a2a53 100644 --- a/salt/mysql/init.sls +++ b/salt/mysql/init.sls @@ -52,8 +52,8 @@ so-mysql: - MYSQL_ROOT_HOST={{ MASTERIP }} - MYSQL_ROOT_PASSWORD=/etc/mypass - binds: - - /opt/so/conf/etc/my.cnf:/etc/my.cnf:ro - - /opt/so/conf/etc/mypass:/etc/mypass + - /opt/so/conf/mysql/etc/my.cnf:/etc/my.cnf:ro + - /opt/so/conf/mysql/etc/mypass:/etc/mypass - /nsm/mysql:/var/lib/mysql:rw - /opt/so/log/mysql:/var/log/mysql:rw - watch: From 8c77d74efe20036e6166455d2a0fb68df8fd026f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 3 Dec 2018 20:29:22 -0500 Subject: [PATCH 007/110] Firewall Module - Add mysql access --- salt/firewall/init.sls | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/salt/firewall/init.sls b/salt/firewall/init.sls index 103172517..a35c01491 100644 --- a/salt/firewall/init.sls +++ b/salt/firewall/init.sls @@ -166,6 +166,17 @@ enable_masternode_influxdb_8086_{{ip}}: - position: 1 - save: True +enable_masternode_mysql_3036_{{ip}}: + iptables.insert: + - table: filter + - chain: DOCKER-USER + - jump: ACCEPT + - proto: tcp + - source: {{ ip }} + - dport: 3306 + - position: 1 + - save: True + {% endfor %} # Make it so all the minions can talk to salt and update etc. From afa39aa7df8975c6dc91b2c4212c81e56d492f61 Mon Sep 17 00:00:00 2001 From: Dustin Lee Date: Mon, 3 Dec 2018 20:36:53 -0500 Subject: [PATCH 008/110] Added basePath for proxy config --- salt/kibana/etc/kibana.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/kibana/etc/kibana.yml b/salt/kibana/etc/kibana.yml index c29218784..bbbfeb575 100644 --- a/salt/kibana/etc/kibana.yml +++ b/salt/kibana/etc/kibana.yml @@ -3,6 +3,7 @@ {%- set ES = salt['pillar.get']('master:mainip', '') -%} server.name: kibana server.host: "0" +server.basePath: /kibana elasticsearch.url: http://{{ ES }}:9200 #elasticsearch.username: elastic #elasticsearch.password: changeme From d911c892baa3acf00b95ac9664533e56b56552cc Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 4 Dec 2018 17:42:50 -0500 Subject: [PATCH 009/110] Setup Script - Restart salt Minion after nuking key --- salt/firewall/init.sls | 2 +- so-setup-network.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/firewall/init.sls b/salt/firewall/init.sls index a35c01491..87c149d99 100644 --- a/salt/firewall/init.sls +++ b/salt/firewall/init.sls @@ -166,7 +166,7 @@ enable_masternode_influxdb_8086_{{ip}}: - position: 1 - save: True -enable_masternode_mysql_3036_{{ip}}: +enable_masternode_mysql_3306_{{ip}}: iptables.insert: - table: filter - chain: DOCKER-USER diff --git a/so-setup-network.sh b/so-setup-network.sh index b44af1c57..21179a18d 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -156,6 +156,7 @@ clear_master() { # This only happens if you re-install the master. if [ -f /etc/salt/pki/minion/minion_master.pub]; then rm /etc/salt/pki/minion/minion_master.pub + service salt-minion restart fi } From 00e57d9b03185de8210b67bc8205cf3c72600994 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 4 Dec 2018 20:24:49 -0500 Subject: [PATCH 010/110] Core Module - Fix font size for PCAP retention so when it hits weeks it looks ok --- .../grafana/grafana_dashboards/forward_nodes/sensor.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/common/grafana/grafana_dashboards/forward_nodes/sensor.json b/salt/common/grafana/grafana_dashboards/forward_nodes/sensor.json index af541d0ec..83a1fc9e6 100644 --- a/salt/common/grafana/grafana_dashboards/forward_nodes/sensor.json +++ b/salt/common/grafana/grafana_dashboards/forward_nodes/sensor.json @@ -1272,7 +1272,7 @@ "thresholds": "259200,432000", "title": "{{ SERVERNAME }} - PCAP Retention", "type": "singlestat", - "valueFontSize": "80%", + "valueFontSize": "70%", "valueMaps": [ { "op": "=", @@ -1280,7 +1280,8 @@ "value": "null" } ], - "valueName": "current" + "valueName": "current", + "decimals": 1 }, { "cacheTimeout": null, From 9a61c460b4deac46807109aa1aa5266760cd9382 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 4 Dec 2018 20:49:26 -0500 Subject: [PATCH 011/110] Core Module - Start Tracking InfluxDB size in telegraf --- salt/common/telegraf/etc/telegraf.conf | 6 ++++-- salt/common/telegraf/scripts/influxdbsize.sh | 5 +++++ 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 salt/common/telegraf/scripts/influxdbsize.sh diff --git a/salt/common/telegraf/etc/telegraf.conf b/salt/common/telegraf/etc/telegraf.conf index 3d1bf30dc..d27d330f3 100644 --- a/salt/common/telegraf/etc/telegraf.conf +++ b/salt/common/telegraf/etc/telegraf.conf @@ -620,7 +620,8 @@ {% if grains['role'] == 'so-master' %} [[inputs.exec]] commands = [ - "/scripts/redis.sh" + "/scripts/redis.sh", + "/scripts/influxdbsize.sh" ] data_format = "influx" {% elif grains['role'] == 'so-SENSOR' %} @@ -642,7 +643,8 @@ "/scripts/suriloss.sh", "/scripts/checkfiles.sh", "/scripts/broloss.sh", - "/scripts/oldpcap.sh" + "/scripts/oldpcap.sh", + "/scripts/influxdbsize.sh" ] data_format = "influx" {% endif %} diff --git a/salt/common/telegraf/scripts/influxdbsize.sh b/salt/common/telegraf/scripts/influxdbsize.sh new file mode 100644 index 000000000..a469da8ae --- /dev/null +++ b/salt/common/telegraf/scripts/influxdbsize.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +INFLUXSIZE=$(du -s -B1 /host/nsm/influxdb | awk {'print $1'} + +echo "influxsize bytes=$INFLUXSIZE" From 0f0adb6c4b4fe2e34ed5dd5d31d5d0a03f4545e5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 4 Dec 2018 20:54:42 -0500 Subject: [PATCH 012/110] Core Module - Changed interval to 30s for telegraf to slow down the amount of data --- salt/common/telegraf/etc/telegraf.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/telegraf/etc/telegraf.conf b/salt/common/telegraf/etc/telegraf.conf index d27d330f3..2f636920b 100644 --- a/salt/common/telegraf/etc/telegraf.conf +++ b/salt/common/telegraf/etc/telegraf.conf @@ -28,7 +28,7 @@ # Configuration for telegraf agent [agent] ## Default data collection interval for all inputs - interval = "10s" + interval = "30s" ## Rounds collection interval to 'interval' ## ie, if interval="10s" then always collect on :00, :10, :20, etc. round_interval = true From 705b242042bf84f93817d6c0d5c2e098d476dd9f Mon Sep 17 00:00:00 2001 From: dlee35 Date: Wed, 5 Dec 2018 08:05:08 -0500 Subject: [PATCH 013/110] Added docker_registry to centos and fixed service restart/enable --- so-setup-network.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/so-setup-network.sh b/so-setup-network.sh index 21179a18d..2d013c813 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -334,6 +334,10 @@ docker_install() { yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo yum -y update yum -y install docker-ce docker-python python-docker + docker_registry + echo "Restarting Docker" + systemctl restart docker + systemctl enable docker else if [ $INSTALLTYPE == 'MASTERONLY' ] || [ $INSTALLTYPE == 'EVALMODE' ]; then From 4c67695cd05b270d39cfdc03d05be1995052d298 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 5 Dec 2018 13:06:29 +0000 Subject: [PATCH 014/110] Setup - Set cur_close_days and log_size_limit --- so-setup-network.sh | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/so-setup-network.sh b/so-setup-network.sh index 21179a18d..5e46412c7 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -22,6 +22,7 @@ NICS=$(ip link | awk -F: '$0 !~ "lo|vir|veth|br|docker|wl|^[^0-9]"{print $2 " \" CPUCORES=$(cat /proc/cpuinfo | grep processor | wc -l) LISTCORES=$(cat /proc/cpuinfo | grep processor | awk '{print $3 " \"" "core" "\""}') RANDOMUID=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 16 | head -n 1) +NODE_ES_PORT="9200" # End Global Variable Section @@ -393,6 +394,16 @@ get_filesystem_nsm(){ FSNSM=$(df /nsm | awk '$3 ~ /[0-9]+/ { print $2 * 1000 }') } +get_log_size_limit() { + + DISK_SIZE_K=`df /nsm |grep -v "^Filesystem" | awk '{print $2}'` + PERCENTAGE=85 + DISK_SIZE=DISK_SIZE_K*1000 + PERCENTAGE_DISK_SPACE=`echo $(($DISK_SIZE*$PERCENTAGE/100))` + LOG_SIZE_LIMIT=$(($PERCENTAGE_DISK_SPACE/1000000000)) + +} + get_filesystem_root(){ FSROOT=$(df / | awk '$3 ~ /[0-9]+/ { print $2 * 1000 }') } @@ -536,6 +547,9 @@ node_pillar() { echo " ls_batch_count: $LSINPUTBATCHCOUNT" >> $TMP/$HOSTNAME.sls echo " es_shard_count: $SHARDCOUNT" >> $TMP/$HOSTNAME.sls echo " node_type: $NODETYPE" >> $TMP/$HOSTNAME.sls + echo " es_port: $NODE_ES_PORT" >> $TMP/$HOSTNAME.sls + echo " log_size_limit: $LOG_SIZE_LIMIT" >> $TMP/$HOSTNAME.sls + echo " cur_close_days: $CURCLOSEDAYS" >> $TMP/$HOSTNAME.sls } @@ -928,6 +942,16 @@ whiptail_check_exitstatus() { } +whiptail_cur_close_days() { + + CURCLOSEDAYS=$(whiptail --title "Security Onion Setup" --inputbox \ + "Please specify the threshold (in days) at which Elasticsearch indices will be closed" 10 60 $CURCLOSEDAYS 3>&1 1>&2 2>&3) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + +} + whiptail_homenet_master() { # Ask for the HOME_NET on the master @@ -975,6 +999,18 @@ whiptail_install_type() { } +whiptail_log_size_limit() { + + LOG_SIZE_LIMIT=$(whiptail --title "Security Onion Setup" --inputbox \ + "Please specify the amount of disk space (in GB) you would like to allocate for Elasticsearch data storage. \ + By default, this is set to 85% of the disk space allotted for /nsm." 10 60 $LOG_SIZE_LIMIT 3>&1 1>&2 2>&3) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + +} + + whiptail_management_nic() { MNIC=$(whiptail --title "NIC Setup" --radiolist "Please select your management NIC" 20 78 12 ${NICS[@]} 3>&1 1>&2 2>&3 ) @@ -1504,11 +1540,13 @@ if (whiptail_you_sure); then NSMSETUP=BASIC NIDS=Suricata BROVERSION=ZEEK + CURCLOSEDAYS=30 whiptail_make_changes clear_master mkdir -p /nsm get_filesystem_root get_filesystem_nsm + get_log_size_limit get_main_ip # Add the user so we can sit back and relax echo "" @@ -1549,6 +1587,10 @@ if (whiptail_you_sure); then whiptail_management_server whiptail_master_updates set_updates + get_log_size_limit + whiptail_log_size_limit + CURCLOSEDAYS=30 + whiptail_cur_close_days es_heapsize ls_heapsize whiptail_node_advanced From ce70e590a1a24c9fa9202b8e6bece68af411caff Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 5 Dec 2018 13:09:07 +0000 Subject: [PATCH 015/110] Curator - break out from ES and add config --- salt/curator/files/action/close.yml | 34 +++++ salt/curator/files/action/delete.yml | 26 ++++ salt/curator/files/bin/so-curator-close | 2 + .../files/bin/so-curator-closed-delete | 41 ++++++ .../files/bin/so-curator-closed-delete-delete | 50 +++++++ salt/curator/files/bin/so-curator-delete | 2 + salt/curator/files/curator.yml | 26 ++++ salt/curator/init.sls | 136 ++++++++++++++++++ 8 files changed, 317 insertions(+) create mode 100644 salt/curator/files/action/close.yml create mode 100644 salt/curator/files/action/delete.yml create mode 100644 salt/curator/files/bin/so-curator-close create mode 100755 salt/curator/files/bin/so-curator-closed-delete create mode 100755 salt/curator/files/bin/so-curator-closed-delete-delete create mode 100644 salt/curator/files/bin/so-curator-delete create mode 100644 salt/curator/files/curator.yml create mode 100644 salt/curator/init.sls diff --git a/salt/curator/files/action/close.yml b/salt/curator/files/action/close.yml new file mode 100644 index 000000000..70d7fa119 --- /dev/null +++ b/salt/curator/files/action/close.yml @@ -0,0 +1,34 @@ +{% if grains['role'] == 'so-node' or grains['role'] == 'so-eval' %} +{%- set cur_close_days = salt['pillar.get']('node:cur_close_days', '') -%} +{%- endif %} + +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close indices older than {{cur_close_days}} days (based on index name), for logstash- + prefixed indices. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: prefix + value: logstash- + exclude: + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/delete.yml b/salt/curator/files/action/delete.yml new file mode 100644 index 000000000..07c19a2ca --- /dev/null +++ b/salt/curator/files/action/delete.yml @@ -0,0 +1,26 @@ +{% if grains['role'] == 'so-node' or grains['role'] == 'so-eval' %} +{%- set log_size_limit = salt['pillar.get']('node:log_size_limit', '') -%} +{%- endif %} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete indices when {{log_size_limit}}(GB) is exceeded. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: prefix + value: logstash- + - filtertype: space + source: creation_date + use_age: True + disk_space: {{log_size_limit}} diff --git a/salt/curator/files/bin/so-curator-close b/salt/curator/files/bin/so-curator-close new file mode 100644 index 000000000..dff6bbb39 --- /dev/null +++ b/salt/curator/files/bin/so-curator-close @@ -0,0 +1,2 @@ +#!/bin/bash +/usr/sbin/so-curator-closed-delete > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/close.yml > /dev/null 2>&1 diff --git a/salt/curator/files/bin/so-curator-closed-delete b/salt/curator/files/bin/so-curator-closed-delete new file mode 100755 index 000000000..000d1cbd0 --- /dev/null +++ b/salt/curator/files/bin/so-curator-closed-delete @@ -0,0 +1,41 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +#. /usr/sbin/so-elastic-common +#. /etc/nsm/securityonion.conf + +# If logrotate script doesn't already exist, create it +#FILE="/etc/logrotate.d/so-curator-closed-delete" +#if ! [ -f ${FILE} ]; then +# cat << EOF > ${FILE} +#/var/log/nsm/so-curator-closed-delete.log { +# daily +# rotate 7 +# copytruncate +# compress +# missingok +# notifempty +#} +#EOF +#fi + +# Avoid starting multiple instances +if pgrep -f "so-curator-closed-delete-delete" >/dev/null; then + echo "Script is already running." +else + /usr/local/bin/so-curator-closed-delete-delete +fi diff --git a/salt/curator/files/bin/so-curator-closed-delete-delete b/salt/curator/files/bin/so-curator-closed-delete-delete new file mode 100755 index 000000000..4b4c5344c --- /dev/null +++ b/salt/curator/files/bin/so-curator-closed-delete-delete @@ -0,0 +1,50 @@ + +{% if grains['role'] == 'so-node' or grains['role'] == 'so-eval' %} +{%- set ELASTICSEARCH_HOST = salt['pillar.get']('node:mainip', '') -%} +{%- set ELASTICSEARCH_PORT = salt['pillar.get']('node:es_port', '') -%} +{%- set LOG_SIZE_LIMIT = salt['pillar.get']('node:log_size_limit', '') -%} +{%- endif %} + +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +#. /usr/sbin/so-elastic-common +#. /etc/nsm/securityonion.conf + +LOG="/opt/so/log/curator/so-curator-closed-delete.log" + +# Check for 2 conditions: +# 1. Are Elasticsearch indices using more disk space than LOG_SIZE_LIMIT? +# 2. Are there any closed logstash- indices that we can delete? +# If both conditions are true, keep on looping until one of the conditions is false. +while [[ $(du -hs --block-size=1GB /nsm/elasticsearch/nodes | awk '{print $1}' ) -gt "{{LOG_SIZE_LIMIT}}" ]] && +curl -s {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices | grep "^ close logstash-" > /dev/null; do + + # We need to determine OLDEST_INDEX. + # First, get the list of closed indices that are prefixed with "logstash-". + # For example: logstash-ids-YYYY.MM.DD + # Then, sort by date by telling sort to use hyphen as delimiter and then sort on the third field. + # Finally, select the first entry in that sorted list. + OLDEST_INDEX=$(curl -s {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices | grep "^ close logstash-" | awk '{print $2}' | sort -t- -k3 | head -1) + + # Now that we've determined OLDEST_INDEX, ask Elasticsearch to delete it. + curl -XDELETE {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/${OLDEST_INDEX} + + # Finally, write a log entry that says we deleted it. + echo "$(date) - Used disk space exceeds LOG_SIZE_LIMIT ({{LOG_SIZE_LIMIT}} GB) - Index ${OLDEST_INDEX} deleted ..." >> ${LOG} + +done diff --git a/salt/curator/files/bin/so-curator-delete b/salt/curator/files/bin/so-curator-delete new file mode 100644 index 000000000..166497855 --- /dev/null +++ b/salt/curator/files/bin/so-curator-delete @@ -0,0 +1,2 @@ +#!/bin/bash +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/delete.yml > /dev/null 2>&1 diff --git a/salt/curator/files/curator.yml b/salt/curator/files/curator.yml new file mode 100644 index 000000000..e115bcd54 --- /dev/null +++ b/salt/curator/files/curator.yml @@ -0,0 +1,26 @@ +{% if grains['role'] == 'so-node' or grains['role'] == 'so-eval' %} +{%- set elasticsearch = salt['pillar.get']('node:mainip', '') -%} +{%- endif %} + +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +client: + hosts: + - {{elasticsearch}} + port: 9200 + url_prefix: + use_ssl: False + certificate: + client_cert: + client_key: + ssl_no_validate: False + http_auth: + timeout: 30 + master_only: False + +logging: + loglevel: INFO + logfile: '/var/log/curator/curator.log' + logformat: default + blacklist: ['elasticsearch', 'urllib3'] diff --git a/salt/curator/init.sls b/salt/curator/init.sls new file mode 100644 index 000000000..d8e3f2817 --- /dev/null +++ b/salt/curator/init.sls @@ -0,0 +1,136 @@ +{% if grains['role'] == 'so-node' or grains['role'] == 'so-eval' %} +# Curator +# Create the group +curatorgroup: + group.present: + - name: curator + - gid: 934 + +# Add user +curator: + user.present: + - uid: 934 + - gid: 934 + - home: /opt/so/conf/curator + - createhome: False + +# Create the log directory +curactiondir: + file.directory: + - name: /opt/so/conf/curator/action + - user: 934 + - group: 939 + - makedirs: True + +curlogdir: + file.directory: + - name: /opt/so/log/curator + - user: 934 + - group: 939 + +curcloseconf: + file.managed: + - name: /opt/so/conf/curator/action/close.yml + - source: salt://curator/files/action/close.yml + - user: 934 + - group: 939 + - template: jinja + +curdelconf: + file.managed: + - name: /opt/so/conf/curator/action/delete.yml + - source: salt://curator/files/action/delete.yml + - user: 934 + - group: 939 + - template: jinja + +curconf: + file.managed: + - name: /opt/so/conf/curator/curator.yml + - source: salt://curator/files/curator.yml + - user: 934 + - group: 939 + - template: jinja + +curcloseddel: + file.managed: + - name: /usr/local/bin/so-curator-closed-delete + - source: salt://curator/files/bin/so-curator-closed-delete + - user: 934 + - group: 939 + - mode: 755 + +curcloseddeldel: + file.managed: + - name: /usr/local/bin/so-curator-closed-delete-delete + - source: salt://curator/files/bin/so-curator-closed-delete-delete + - user: 934 + - group: 939 + - mode: 755 + - template: jinja + +curclose: + file.managed: + - name: /usr/local/bin/so-curator-close + - source: salt://curator/files/bin/so-curator-close + - user: 934 + - group: 939 + - mode: 755 + +curdel: + file.managed: + - name: /usr/local/bin/so-curator-delete + - source: salt://curator/files/bin/so-curator-delete + - user: 934 + - group: 939 + - mode: 755 + +/usr/local/bin/so-curator-closed-delete: + cron.present: + - user: root + - minute: '*' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' + +/usr/local/bin/so-curator-close: + cron.present: + - user: root + - minute: '*' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' + +/usr/local/bin/so-curator-delete: + cron.present: + - user: root + - minute: '*' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' + + +so-curator: + docker_container.running: + - image: soshybridhunter/so-curator:HH1.0.3 + - hostname: curator + - name: so-curator + - user: curator + - interactive: True + - tty: True + - binds: + - /opt/so/conf/curator/curator.yml:/etc/curator/config/curator.yml:ro + - /opt/so/conf/curator/action/:/etc/curator/action:ro + - /opt/so/log/curator:/var/log/curator:rw +# Begin Curator Cron Jobs + +# Close +# Delete +# Hot Warm +# Segment Merge + +# End Curator Cron Jobs +{% endif %} From 1ccece70efebb4c6829aa74d887fd4a417930f1b Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 5 Dec 2018 13:10:29 +0000 Subject: [PATCH 016/110] Curator - Remove config from ES --- salt/elasticsearch/init.sls | 81 ++----------------------------------- 1 file changed, 3 insertions(+), 78 deletions(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 75cc7e5f7..122e2f7eb 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -150,6 +150,7 @@ so-freq: docker_container.running: - image: soshybridhunter/so-freqserver:HH1.0.3 - hostname: freqserver + - name: so-freqserver - user: freqserver - binds: - /opt/so/log/freq_server:/var/log/freq_server:rw @@ -185,7 +186,7 @@ so-domainstats: docker_container.running: - image: soshybridhunter/so-domainstats:HH1.0.3 - hostname: domainstats - - name: domainstats + - name: so-domainstats - user: domainstats - binds: - /opt/so/log/domainstats:/var/log/domain_stats @@ -193,82 +194,6 @@ so-domainstats: {% endif %} -# Curator -# Create the group -curatorgroup: - group.present: - - name: curator - - gid: 934 - -# Add user -curator: - user.present: - - uid: 934 - - gid: 934 - - home: /opt/so/conf/curator - - createhome: False - -# Create the log directory -curactiondir: - file.directory: - - name: /opt/so/conf/curator/action - - user: 934 - - group: 939 - - makedirs: True - -curlogdir: - file.directory: - - name: /opt/so/log/curator - - user: 934 - - group: 939 - -curclose: - file.managed: - - name: /opt/so/conf/curator/action/close.yml - - source: salt://elasticsearch/files/curator/action/close.yml - - user: 934 - - group: 939 - - template: jinja - -curdel: - file.managed: - - name: /opt/so/conf/curator/action/delete.yml - - source: salt://elasticsearch/files/curator/action/delete.yml - - user: 934 - - group: 939 - - template: jinja - -curconf: - file.managed: - - name: /opt/so/conf/curator/curator.yml - - source: salt://elasticsearch/files/curator/curator.yml - - user: 934 - - group: 939 - - template: jinja - -so-curator: - docker_container.running: - - image: soshybridhunter/so-curator:HH1.0.3 - - hostname: curator - - name: curator - - user: curator - - interactive: True - - tty: True - - binds: - - /opt/so/conf/curator/curator.yml:/etc/curator/config/curator.yml:ro - - /opt/so/conf/curator/action/:/etc/curator/action:ro - - /opt/so/log/curator:/var/log/curator:rw - - -# Begin Curator Cron Jobs - -# Close -# Delete -# Hot Warm -# Segment Merge - -# End Curator Cron Jobs - # Elastalert {% if esalert == 1 %} @@ -311,7 +236,7 @@ so-elastalert: docker_container.running: - image: soshybridhunter/so-elastalert:HH1.0.3 - hostname: elastalert - - name: elastalert + - name: so-elastalert - user: elastalert - detach: True - binds: From 2e73a6fb0219fd0d62bae9eb02489da4deabc080 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 5 Dec 2018 13:13:15 +0000 Subject: [PATCH 017/110] Curator - Update top.sls for Curator --- salt/top.sls | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/top.sls b/salt/top.sls index 1ab76733e..0b2c50ca4 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -27,8 +27,9 @@ base: - pcap - suricata - bro + - curator - utility - - schedule + - schedule 'G@role:so-master': @@ -60,6 +61,7 @@ base: - firewall - logstash - elasticsearch + - curator - schedule 'G@role:so-node and I@node:node_type:warm': @@ -77,6 +79,7 @@ base: - firewall - logstash - elasticsearch + - curator - schedule 'G@role:mastersensor': From 90e3b6912ce1c70be63d5c948ff1372f4dfa3940 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 5 Dec 2018 13:14:19 +0000 Subject: [PATCH 018/110] Curator - Remove Curator files from ES module --- .../files/curator/action/close.yml | 30 ------------------- .../files/curator/action/delete.yml | 23 -------------- salt/elasticsearch/files/curator/curator.yml | 22 -------------- 3 files changed, 75 deletions(-) delete mode 100644 salt/elasticsearch/files/curator/action/close.yml delete mode 100644 salt/elasticsearch/files/curator/action/delete.yml delete mode 100644 salt/elasticsearch/files/curator/curator.yml diff --git a/salt/elasticsearch/files/curator/action/close.yml b/salt/elasticsearch/files/curator/action/close.yml deleted file mode 100644 index a9ca54335..000000000 --- a/salt/elasticsearch/files/curator/action/close.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- -# Remember, leave a key empty if there is no value. None will be a string, -# not a Python "NoneType" -# -# Also remember that all examples have 'disable_action' set to True. If you -# want to use this action as a template, be sure to set this to False after -# copying it. -actions: - 1: - action: close - description: >- - Close indices older than 2 days (based on index name), for logstash- - prefixed indices. - options: - delete_aliases: False - timeout_override: - continue_if_exception: False - disable_action: False - filters: - - filtertype: pattern - kind: prefix - value: logstash- - exclude: - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: 2 - exclude: diff --git a/salt/elasticsearch/files/curator/action/delete.yml b/salt/elasticsearch/files/curator/action/delete.yml deleted file mode 100644 index cb55ec0c3..000000000 --- a/salt/elasticsearch/files/curator/action/delete.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -# Remember, leave a key empty if there is no value. None will be a string, -# not a Python "NoneType" -# -# Also remember that all examples have 'disable_action' set to True. If you -# want to use this action as a template, be sure to set this to False after -# copying it. -actions: - 1: - action: delete_indices - description: >- - Delete indices when $disk_space value (in GB) is exceeded. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: prefix - value: logstash- - - filtertype: space - source: creation_date - use_age: True - disk_space: 43 diff --git a/salt/elasticsearch/files/curator/curator.yml b/salt/elasticsearch/files/curator/curator.yml deleted file mode 100644 index dc8b69309..000000000 --- a/salt/elasticsearch/files/curator/curator.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -# Remember, leave a key empty if there is no value. None will be a string, -# not a Python "NoneType" -client: - hosts: - - elasticsearch - port: 9200 - url_prefix: - use_ssl: False - certificate: - client_cert: - client_key: - ssl_no_validate: False - http_auth: - timeout: 30 - master_only: False - -logging: - loglevel: INFO - logfile: '/var/log/curator/curator.log' - logformat: default - blacklist: ['elasticsearch', 'urllib3'] From 7a260a4876cff7b36a537567a136e78d14c7c4f1 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 5 Dec 2018 13:24:16 +0000 Subject: [PATCH 019/110] Curator - Move files to /usr/sbin --- salt/curator/files/bin/so-curator-closed-delete | 2 +- salt/curator/init.sls | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/salt/curator/files/bin/so-curator-closed-delete b/salt/curator/files/bin/so-curator-closed-delete index 000d1cbd0..4382a721d 100755 --- a/salt/curator/files/bin/so-curator-closed-delete +++ b/salt/curator/files/bin/so-curator-closed-delete @@ -37,5 +37,5 @@ if pgrep -f "so-curator-closed-delete-delete" >/dev/null; then echo "Script is already running." else - /usr/local/bin/so-curator-closed-delete-delete + /usr/sbin/so-curator-closed-delete-delete fi diff --git a/salt/curator/init.sls b/salt/curator/init.sls index d8e3f2817..adbf4f6b0 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -54,7 +54,7 @@ curconf: curcloseddel: file.managed: - - name: /usr/local/bin/so-curator-closed-delete + - name: /usr/sbin/so-curator-closed-delete - source: salt://curator/files/bin/so-curator-closed-delete - user: 934 - group: 939 @@ -62,7 +62,7 @@ curcloseddel: curcloseddeldel: file.managed: - - name: /usr/local/bin/so-curator-closed-delete-delete + - name: /usr/sbin/so-curator-closed-delete-delete - source: salt://curator/files/bin/so-curator-closed-delete-delete - user: 934 - group: 939 @@ -71,7 +71,7 @@ curcloseddeldel: curclose: file.managed: - - name: /usr/local/bin/so-curator-close + - name: /usr/sbin/so-curator-close - source: salt://curator/files/bin/so-curator-close - user: 934 - group: 939 @@ -79,13 +79,13 @@ curclose: curdel: file.managed: - - name: /usr/local/bin/so-curator-delete + - name: /usr/sbin/so-curator-delete - source: salt://curator/files/bin/so-curator-delete - user: 934 - group: 939 - mode: 755 -/usr/local/bin/so-curator-closed-delete: +/usr/sbin/so-curator-closed-delete: cron.present: - user: root - minute: '*' @@ -94,7 +94,7 @@ curdel: - month: '*' - dayweek: '*' -/usr/local/bin/so-curator-close: +/usr/sbin/so-curator-close: cron.present: - user: root - minute: '*' @@ -103,7 +103,7 @@ curdel: - month: '*' - dayweek: '*' -/usr/local/bin/so-curator-delete: +/usr/sbin/so-curator-delete: cron.present: - user: root - minute: '*' From 3461c7e22608090ecb301adb4ca94eef248c855b Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 5 Dec 2018 15:14:36 +0000 Subject: [PATCH 020/110] Curator - Update config for Eval Mode --- salt/curator/files/action/close.yml | 8 +++++++- salt/curator/files/action/delete.yml | 8 +++++++- salt/curator/files/bin/so-curator-closed-delete-delete | 10 +++++++++- salt/curator/files/curator.yml | 8 +++++++- 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/salt/curator/files/action/close.yml b/salt/curator/files/action/close.yml index 70d7fa119..ab4d07050 100644 --- a/salt/curator/files/action/close.yml +++ b/salt/curator/files/action/close.yml @@ -1,5 +1,11 @@ -{% if grains['role'] == 'so-node' or grains['role'] == 'so-eval' %} +{% if grains['role'] == 'so-node' %} + {%- set cur_close_days = salt['pillar.get']('node:cur_close_days', '') -%} + +{% elif grains['role'] == 'so-eval' %} + +{%- set cur_close_days = salt['pillar.get']('master:cur_close_days', '') -%} + {%- endif %} --- diff --git a/salt/curator/files/action/delete.yml b/salt/curator/files/action/delete.yml index 07c19a2ca..1b3440f8b 100644 --- a/salt/curator/files/action/delete.yml +++ b/salt/curator/files/action/delete.yml @@ -1,5 +1,11 @@ -{% if grains['role'] == 'so-node' or grains['role'] == 'so-eval' %} +{% if grains['role'] == 'so-node' %} + {%- set log_size_limit = salt['pillar.get']('node:log_size_limit', '') -%} + +{% elif grains['role'] == 'so-eval' %} + +{%- set log_size_limit = salt['pillar.get']('master:log_size_limit', '') -%} + {%- endif %} --- # Remember, leave a key empty if there is no value. None will be a string, diff --git a/salt/curator/files/bin/so-curator-closed-delete-delete b/salt/curator/files/bin/so-curator-closed-delete-delete index 4b4c5344c..8841c843f 100755 --- a/salt/curator/files/bin/so-curator-closed-delete-delete +++ b/salt/curator/files/bin/so-curator-closed-delete-delete @@ -1,8 +1,16 @@ -{% if grains['role'] == 'so-node' or grains['role'] == 'so-eval' %} +{% if grains['role'] == 'so-node' %} + {%- set ELASTICSEARCH_HOST = salt['pillar.get']('node:mainip', '') -%} {%- set ELASTICSEARCH_PORT = salt['pillar.get']('node:es_port', '') -%} {%- set LOG_SIZE_LIMIT = salt['pillar.get']('node:log_size_limit', '') -%} + +{% elif grains['role'] == 'so-eval' %} + +{%- set ELASTICSEARCH_HOST = salt['pillar.get']('master:mainip', '') -%} +{%- set ELASTICSEARCH_PORT = salt['pillar.get']('master:es_port', '') -%} +{%- set LOG_SIZE_LIMIT = salt['pillar.get']('master:log_size_limit', '') -%} + {%- endif %} #!/bin/bash diff --git a/salt/curator/files/curator.yml b/salt/curator/files/curator.yml index e115bcd54..aa55836b1 100644 --- a/salt/curator/files/curator.yml +++ b/salt/curator/files/curator.yml @@ -1,5 +1,11 @@ -{% if grains['role'] == 'so-node' or grains['role'] == 'so-eval' %} +{% if grains['role'] == 'so-node' %} + {%- set elasticsearch = salt['pillar.get']('node:mainip', '') -%} + +{% elif grains['role'] == 'so-eval' %} + +{%- set elasticsearch = salt['pillar.get']('master:mainip', '') -%} + {%- endif %} --- From aa32a55b4edaa2b67fa9e7d2cf8cc4eccaf58e49 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 5 Dec 2018 15:16:14 +0000 Subject: [PATCH 021/110] Curator - Update Master pillar in setup --- so-setup-network.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/so-setup-network.sh b/so-setup-network.sh index 5e46412c7..e26bbb163 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -500,6 +500,9 @@ master_pillar() { echo " oinkcode: $OINKCODE" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls #echo " access_key: $ACCESS_KEY" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls #echo " access_secret: $ACCESS_SECRET" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls + echo " es_port: $NODE_ES_PORT" >> $TMP/$HOSTNAME.sls + echo " log_size_limit: $LOG_SIZE_LIMIT" >> $TMP/$HOSTNAME.sls + echo " cur_close_days: $CURCLOSEDAYS" >> $TMP/$HOSTNAME.sls } From 88c144a250b7d2b6cc07b2ef75cba11cf2bc0dc0 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 5 Dec 2018 15:51:10 +0000 Subject: [PATCH 022/110] Curator - Fix path for master pillar additions --- so-setup-network.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index e26bbb163..b4a19943b 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -500,9 +500,9 @@ master_pillar() { echo " oinkcode: $OINKCODE" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls #echo " access_key: $ACCESS_KEY" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls #echo " access_secret: $ACCESS_SECRET" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls - echo " es_port: $NODE_ES_PORT" >> $TMP/$HOSTNAME.sls - echo " log_size_limit: $LOG_SIZE_LIMIT" >> $TMP/$HOSTNAME.sls - echo " cur_close_days: $CURCLOSEDAYS" >> $TMP/$HOSTNAME.sls + echo " es_port: $NODE_ES_PORT" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls + echo " log_size_limit: $LOG_SIZE_LIMIT" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls + echo " cur_close_days: $CURCLOSEDAYS" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls } From 51ffa32e6769f86d258e178fae5733b2f1d801c6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 5 Dec 2018 11:09:40 -0500 Subject: [PATCH 023/110] MySQL Module - Change name to socore --- salt/mysql/etc/my.cnf | 2 +- salt/mysql/init.sls | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/salt/mysql/etc/my.cnf b/salt/mysql/etc/my.cnf index e1bf4e117..e37f690dc 100644 --- a/salt/mysql/etc/my.cnf +++ b/salt/mysql/etc/my.cnf @@ -22,7 +22,7 @@ skip-name-resolve datadir=/var/lib/mysql socket=/var/lib/mysql/mysql.sock secure-file-priv=/var/lib/mysql-files -user=939 +user=socore # Disabling symbolic-links is recommended to prevent assorted security risks symbolic-links=0 diff --git a/salt/mysql/init.sls b/salt/mysql/init.sls index 1683a2a53..179727a5a 100644 --- a/salt/mysql/init.sls +++ b/salt/mysql/init.sls @@ -19,6 +19,13 @@ mysqletcdir: - group: 939 - makedirs: True +mysqlpiddir: + file.directory: + - name: /opt/so/conf/mysql/pid + - user: 939 + - group: 939 + - makedirs: True + lsetcsync: file.recurse: - name: /opt/so/conf/mysql/etc From 314e35b2be1eb5ac64ee0b9a9eba44fc6b14112d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 5 Dec 2018 11:10:34 -0500 Subject: [PATCH 024/110] Kibana Module - Added Dustin's dashboard mods --- salt/kibana/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/init.sls b/salt/kibana/init.sls index b4b641862..4bd800297 100644 --- a/salt/kibana/init.sls +++ b/salt/kibana/init.sls @@ -59,7 +59,7 @@ synckibanacustom: # Start the kibana docker so-kibana: docker_container.running: - - image: soshybridhunter/so-kibana:HH1.0.3 + - image: soshybridhunter/so-kibana:HH1.0.5 - hostname: kibana - user: kibana - environment: From 658bc2cc8e604e0bdf126618e342cb01b49e9934 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 5 Dec 2018 12:43:13 -0500 Subject: [PATCH 025/110] Logstash Module - Update Version --- salt/logstash/init.sls | 2 +- salt/mysql/init.sls | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 6901a857f..e816c888e 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -149,7 +149,7 @@ lslogdir: so-logstash: docker_container.running: - - image: soshybridhunter/so-logstash:HH1.0.4 + - image: soshybridhunter/so-logstash:HH1.0.5 - hostname: so-logstash - name: so-logstash - user: logstash diff --git a/salt/mysql/init.sls b/salt/mysql/init.sls index 179727a5a..e48e8106d 100644 --- a/salt/mysql/init.sls +++ b/salt/mysql/init.sls @@ -57,7 +57,7 @@ so-mysql: - 0.0.0.0:3306:3306 - environment: - MYSQL_ROOT_HOST={{ MASTERIP }} - - MYSQL_ROOT_PASSWORD=/etc/mypass + - MYSQL_ROOT_PASSWORD={{ MYSQLPASS }} - binds: - /opt/so/conf/mysql/etc/my.cnf:/etc/my.cnf:ro - /opt/so/conf/mysql/etc/mypass:/etc/mypass From 7fc1f576d99c17e08f228093ce51eacf636248dc Mon Sep 17 00:00:00 2001 From: dlee35 Date: Wed, 5 Dec 2018 15:05:11 -0500 Subject: [PATCH 026/110] changed tag eval to type --- salt/logstash/files/dynamic/0006_input_beats.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/logstash/files/dynamic/0006_input_beats.conf b/salt/logstash/files/dynamic/0006_input_beats.conf index d3615b126..b263e611b 100644 --- a/salt/logstash/files/dynamic/0006_input_beats.conf +++ b/salt/logstash/files/dynamic/0006_input_beats.conf @@ -9,7 +9,7 @@ input { } } filter { - if "ids" in [tags] { + if [type] == "ids" { mutate { rename => { "host" => "beat_host" } remove_tag => ["beat"] From b22b10926d92d20cb0cff2ebc2d21e09930b1303 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 5 Dec 2018 15:12:53 -0500 Subject: [PATCH 027/110] MySQL Module - Fix password file formatting --- salt/mysql/etc/mypass | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/mysql/etc/mypass b/salt/mysql/etc/mypass index 723d4722f..dad35b5e3 100644 --- a/salt/mysql/etc/mypass +++ b/salt/mysql/etc/mypass @@ -1,2 +1,2 @@ -{%- set MYSQLPASS = salt['pillar.get']('master:mysqlpass', 'iwonttellyou') %} +{%- set MYSQLPASS = salt['pillar.get']('master:mysqlpass', 'iwonttellyou') -%} {{ MYSQLPASS }} From 53f7fcd07c8aa25b3993d4f7c41181b2e549e985 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 5 Dec 2018 15:54:43 -0500 Subject: [PATCH 028/110] Fleet Module - SSL additions --- salt/ca/files/signing_policies.conf | 13 +++++++++++++ salt/fleet/init.sls | 24 ++++++++++++++++++++++++ salt/ssl/init.sls | 14 ++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/salt/ca/files/signing_policies.conf b/salt/ca/files/signing_policies.conf index 04724ef70..a6ecdd4c3 100644 --- a/salt/ca/files/signing_policies.conf +++ b/salt/ca/files/signing_policies.conf @@ -51,3 +51,16 @@ x509_signing_policies: - authorityKeyIdentifier: keyid,issuer:always - days_valid: 3000 - copypath: /etc/pki/issued_certs/ + fleet: + - minions: '*' + - signing_private_key: /etc/pki/ca.key + - signing_cert: /etc/pki/ca.crt + - C: US + - ST: Utah + - L: Salt Lake City + - basicConstraints: "critical CA:false" + - keyUsage: "critical keyEncipherment" + - subjectKeyIdentifier: hash + - authorityKeyIdentifier: keyid,issuer:always + - days_valid: 3000 + - copypath: /etc/pki/issued_certs/ diff --git a/salt/fleet/init.sls b/salt/fleet/init.sls index abb1828c8..5dbcea390 100644 --- a/salt/fleet/init.sls +++ b/salt/fleet/init.sls @@ -1,3 +1,6 @@ +{%- set FLEETPASS = salt['pillar.get']('master:fleetpass', 'bazinga') -%} +{%- set MASTERIP = salt['pillar.get']('static:masterip', '') -%} + # Fleet Setup fleetcdir: file.directory: @@ -5,3 +8,24 @@ fleetcdir: - user: 939 - group: 939 - makedirs: True + +so-fleet: + docker_container.running: + - image: kolide/fleet + - hostname: so-fleet + - user: socore + - port_bindings: + - 0.0.0.0:8080:8080 + - environment: + - KOLIDE_MYSQL_ADDRESS={{ MASTERIP }}:3306 + - KOLIDE_MYSQL_DATABASE=fleet + - KOLIDE_MYSQL_USERNAME=fleetdbuser + - KOLIDE_MYSQL_PASSWORD={{ FLEETPASS }} + - KOLIDE_REDIS_ADDRESS={{ MASTERIP }}:6379 + - KOLIDE_SERVER_CERT=/tmp/server.cert + - KOLIDE_SERVER_KEY=/tmp/server.key + - KOLIDE_LOGGING_JSON=true + - binds: + - /opt/so/conf/fleet/etc:/ssl:ro + - watch: + - /opt/so/conf/fleet/etc diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 8a8d9309a..12ab3a9c7 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -92,6 +92,20 @@ fbcrtlink: bits: 4096 backup: True +# Create a cert for OSQuery +/etc/pki/fleet.crt: + x509.certificate_managed: + - ca_server: {{ master }} + - signing_policy: fleet + - public_key: /etc/pki/fleet.key + - CN: {{ master }} + - days_remaining: 3000 + - backup: True + - managed_private_key: + name: /etc/pki/fleet.key + bits: 4096 + backup: True + {% endif %} {% if grains['role'] == 'so-SENSOR' or grains['role'] == 'so-eval' %} From f988c7c0e74b1285cc4a636818e9b7c03e17f507 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 5 Dec 2018 17:16:46 -0500 Subject: [PATCH 029/110] Fleet Module - SSL additions --- salt/fleet/init.sls | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/fleet/init.sls b/salt/fleet/init.sls index 5dbcea390..47e0cab30 100644 --- a/salt/fleet/init.sls +++ b/salt/fleet/init.sls @@ -22,10 +22,11 @@ so-fleet: - KOLIDE_MYSQL_USERNAME=fleetdbuser - KOLIDE_MYSQL_PASSWORD={{ FLEETPASS }} - KOLIDE_REDIS_ADDRESS={{ MASTERIP }}:6379 - - KOLIDE_SERVER_CERT=/tmp/server.cert - - KOLIDE_SERVER_KEY=/tmp/server.key + - KOLIDE_SERVER_CERT=/ssl/server.cert + - KOLIDE_SERVER_KEY=/ssl/server.key - KOLIDE_LOGGING_JSON=true - binds: - - /opt/so/conf/fleet/etc:/ssl:ro + - /etc/pki/fleet.key:/ssl/server.key:ro + - /etc/pki/fleet.crt:/ssl/server.cert - watch: - /opt/so/conf/fleet/etc From b2c711af537266144e731b38b9bc180fde27d641 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 5 Dec 2018 17:21:46 -0500 Subject: [PATCH 030/110] MYSQL Module - Modify network for fleetdb user --- salt/fleet/init.sls | 1 - salt/mysql/init.sls | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/fleet/init.sls b/salt/fleet/init.sls index 47e0cab30..d3ae946e8 100644 --- a/salt/fleet/init.sls +++ b/salt/fleet/init.sls @@ -13,7 +13,6 @@ so-fleet: docker_container.running: - image: kolide/fleet - hostname: so-fleet - - user: socore - port_bindings: - 0.0.0.0:8080:8080 - environment: diff --git a/salt/mysql/init.sls b/salt/mysql/init.sls index e48e8106d..a2536851f 100644 --- a/salt/mysql/init.sls +++ b/salt/mysql/init.sls @@ -72,7 +72,7 @@ fleetdb: fleetdbuser: mysql_user.present: - - host: {{ MASTERIP }} + - host: 172.17.0.0/255.255.0.0 - password: {{ FLEETPASS }} - connection_user: root - connection_pass: {{ MYSQLPASS }} From e9d97286c369e19321126d37542a2b58850c12ad Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 5 Dec 2018 18:29:46 -0500 Subject: [PATCH 031/110] MYSQL Module - Modify network for fleetdb user --- salt/mysql/init.sls | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/mysql/init.sls b/salt/mysql/init.sls index a2536851f..36faf9675 100644 --- a/salt/mysql/init.sls +++ b/salt/mysql/init.sls @@ -57,7 +57,7 @@ so-mysql: - 0.0.0.0:3306:3306 - environment: - MYSQL_ROOT_HOST={{ MASTERIP }} - - MYSQL_ROOT_PASSWORD={{ MYSQLPASS }} + - MYSQL_ROOT_PASSWORD=/etc/mypass - binds: - /opt/so/conf/mysql/etc/my.cnf:/etc/my.cnf:ro - /opt/so/conf/mysql/etc/mypass:/etc/mypass @@ -81,4 +81,5 @@ fleetdbpriv: mysql_grants.present: - grant: all privileges - database: fleet.* - - user: fleet + - user: fleetdbuser + - host: 172.17.0.0/255.255.0.0 From 24cb7f4a1481cadc847a788544c5381cf4311ad8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 5 Dec 2018 19:50:59 -0500 Subject: [PATCH 032/110] Fleet Modules --- salt/firewall/init.sls | 11 +++++++++++ salt/fleet/init.sls | 1 + 2 files changed, 12 insertions(+) diff --git a/salt/firewall/init.sls b/salt/firewall/init.sls index 87c149d99..71575e3d6 100644 --- a/salt/firewall/init.sls +++ b/salt/firewall/init.sls @@ -357,6 +357,17 @@ enable_standard_analyst_5601_{{ip}}: - dport: 5601 - position: 1 - save: True +#THIS IS TEMPORARY +enable_standard_analyst_8080_{{ip}}: + iptables.insert: + - table: filter + - chain: DOCKER-USER + - jump: ACCEPT + - proto: tcp + - source: {{ ip }} + - dport: 8080 + - position: 1 + - save: True {% endfor %} diff --git a/salt/fleet/init.sls b/salt/fleet/init.sls index d3ae946e8..59c02c0ce 100644 --- a/salt/fleet/init.sls +++ b/salt/fleet/init.sls @@ -24,6 +24,7 @@ so-fleet: - KOLIDE_SERVER_CERT=/ssl/server.cert - KOLIDE_SERVER_KEY=/ssl/server.key - KOLIDE_LOGGING_JSON=true + - KOLIDE_AUTH_JWT_KEY=thisisatest - binds: - /etc/pki/fleet.key:/ssl/server.key:ro - /etc/pki/fleet.crt:/ssl/server.cert From 5c2857667444771ed36568adbe265de076f7d9cf Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 6 Dec 2018 02:38:59 +0000 Subject: [PATCH 033/110] Logtash - Update input for Eval Mode --- salt/logstash/conf/conf.enabled.txt.so-eval | 2 +- .../conf/conf.enabled.txt.so-eval.old | 109 ++++++++++++++++++ salt/top.sls | 1 + 3 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 salt/logstash/conf/conf.enabled.txt.so-eval.old diff --git a/salt/logstash/conf/conf.enabled.txt.so-eval b/salt/logstash/conf/conf.enabled.txt.so-eval index e5ce9c803..71e50525f 100644 --- a/salt/logstash/conf/conf.enabled.txt.so-eval +++ b/salt/logstash/conf/conf.enabled.txt.so-eval @@ -12,8 +12,8 @@ /usr/share/logstash/pipeline.so/0002_input_windows_json.conf /usr/share/logstash/pipeline.so/0003_input_syslog.conf /usr/share/logstash/pipeline.so/0005_input_suricata.conf +/usr/share/logstash/pipeline.dynamic/0006_input_beats.conf /usr/share/logstash/pipeline.so/0007_input_import.conf -/usr/share/logstash/pipeline.so/0008_input_eval.conf /usr/share/logstash/pipeline.so/1000_preprocess_log_elapsed.conf /usr/share/logstash/pipeline.so/1001_preprocess_syslogng.conf /usr/share/logstash/pipeline.so/1002_preprocess_json.conf diff --git a/salt/logstash/conf/conf.enabled.txt.so-eval.old b/salt/logstash/conf/conf.enabled.txt.so-eval.old new file mode 100644 index 000000000..e5ce9c803 --- /dev/null +++ b/salt/logstash/conf/conf.enabled.txt.so-eval.old @@ -0,0 +1,109 @@ +# This is where can specify which LogStash configs get loaded. +# +# The custom folder on the master gets automatically synced to each logstash +# node. +# +# To enable a custom configuration see the following example and uncomment: +# /usr/share/logstash/pipeline.custom/1234_input_custom.conf +## +# All of the defaults are loaded. +/usr/share/logstash/pipeline.so/0000_input_syslogng.conf +/usr/share/logstash/pipeline.so/0001_input_json.conf +/usr/share/logstash/pipeline.so/0002_input_windows_json.conf +/usr/share/logstash/pipeline.so/0003_input_syslog.conf +/usr/share/logstash/pipeline.so/0005_input_suricata.conf +/usr/share/logstash/pipeline.so/0007_input_import.conf +/usr/share/logstash/pipeline.so/0008_input_eval.conf +/usr/share/logstash/pipeline.so/1000_preprocess_log_elapsed.conf +/usr/share/logstash/pipeline.so/1001_preprocess_syslogng.conf +/usr/share/logstash/pipeline.so/1002_preprocess_json.conf +/usr/share/logstash/pipeline.so/1003_preprocess_bro.conf +/usr/share/logstash/pipeline.so/1004_preprocess_syslog_types.conf +/usr/share/logstash/pipeline.so/1026_preprocess_dhcp.conf +/usr/share/logstash/pipeline.so/1029_preprocess_esxi.conf +/usr/share/logstash/pipeline.so/1030_preprocess_greensql.conf +/usr/share/logstash/pipeline.so/1031_preprocess_iis.conf +/usr/share/logstash/pipeline.so/1032_preprocess_mcafee.conf +/usr/share/logstash/pipeline.so/1033_preprocess_snort.conf +/usr/share/logstash/pipeline.so/1034_preprocess_syslog.conf +/usr/share/logstash/pipeline.so/1100_preprocess_bro_conn.conf +/usr/share/logstash/pipeline.so/1101_preprocess_bro_dhcp.conf +/usr/share/logstash/pipeline.so/1102_preprocess_bro_dns.conf +/usr/share/logstash/pipeline.so/1103_preprocess_bro_dpd.conf +/usr/share/logstash/pipeline.so/1104_preprocess_bro_files.conf +/usr/share/logstash/pipeline.so/1105_preprocess_bro_ftp.conf +/usr/share/logstash/pipeline.so/1106_preprocess_bro_http.conf +/usr/share/logstash/pipeline.so/1107_preprocess_bro_irc.conf +/usr/share/logstash/pipeline.so/1108_preprocess_bro_kerberos.conf +/usr/share/logstash/pipeline.so/1109_preprocess_bro_notice.conf +/usr/share/logstash/pipeline.so/1110_preprocess_bro_rdp.conf +/usr/share/logstash/pipeline.so/1111_preprocess_bro_signatures.conf +/usr/share/logstash/pipeline.so/1112_preprocess_bro_smtp.conf +/usr/share/logstash/pipeline.so/1113_preprocess_bro_snmp.conf +/usr/share/logstash/pipeline.so/1114_preprocess_bro_software.conf +/usr/share/logstash/pipeline.so/1115_preprocess_bro_ssh.conf +/usr/share/logstash/pipeline.so/1116_preprocess_bro_ssl.conf +/usr/share/logstash/pipeline.so/1117_preprocess_bro_syslog.conf +/usr/share/logstash/pipeline.so/1118_preprocess_bro_tunnel.conf +/usr/share/logstash/pipeline.so/1119_preprocess_bro_weird.conf +/usr/share/logstash/pipeline.so/1121_preprocess_bro_mysql.conf +/usr/share/logstash/pipeline.so/1122_preprocess_bro_socks.conf +/usr/share/logstash/pipeline.so/1123_preprocess_bro_x509.conf +/usr/share/logstash/pipeline.so/1124_preprocess_bro_intel.conf +/usr/share/logstash/pipeline.so/1125_preprocess_bro_modbus.conf +/usr/share/logstash/pipeline.so/1126_preprocess_bro_sip.conf +/usr/share/logstash/pipeline.so/1127_preprocess_bro_radius.conf +/usr/share/logstash/pipeline.so/1128_preprocess_bro_pe.conf +/usr/share/logstash/pipeline.so/1129_preprocess_bro_rfb.conf +/usr/share/logstash/pipeline.so/1130_preprocess_bro_dnp3.conf +/usr/share/logstash/pipeline.so/1131_preprocess_bro_smb_files.conf +/usr/share/logstash/pipeline.so/1132_preprocess_bro_smb_mapping.conf +/usr/share/logstash/pipeline.so/1133_preprocess_bro_ntlm.conf +/usr/share/logstash/pipeline.so/1134_preprocess_bro_dce_rpc.conf +/usr/share/logstash/pipeline.so/1998_test_data.conf +/usr/share/logstash/pipeline.so/2000_network_flow.conf +/usr/share/logstash/pipeline.so/6000_bro.conf +/usr/share/logstash/pipeline.so/6001_bro_import.conf +/usr/share/logstash/pipeline.so/6002_syslog.conf +/usr/share/logstash/pipeline.so/6101_switch_brocade.conf +/usr/share/logstash/pipeline.so/6200_firewall_fortinet.conf +/usr/share/logstash/pipeline.so/6201_firewall_pfsense.conf +/usr/share/logstash/pipeline.so/6300_windows.conf +/usr/share/logstash/pipeline.so/6301_dns_windows.conf +/usr/share/logstash/pipeline.so/6400_suricata.conf +/usr/share/logstash/pipeline.so/6500_ossec.conf +/usr/share/logstash/pipeline.so/6501_ossec_sysmon.conf +/usr/share/logstash/pipeline.so/6502_ossec_autoruns.conf +/usr/share/logstash/pipeline.so/6600_winlogbeat_sysmon.conf +/usr/share/logstash/pipeline.so/6700_winlogbeat.conf +/usr/share/logstash/pipeline.so/8000_postprocess_bro_cleanup.conf +/usr/share/logstash/pipeline.so/8001_postprocess_common_ip_augmentation.conf +#/usr/share/logstash/pipeline.so/8006_postprocess_dns.conf +#/usr/share/logstash/pipeline.so/8007_postprocess_dns_top1m_tagging.conf +/usr/share/logstash/pipeline.so/8007_postprocess_http.conf +#/usr/share/logstash/pipeline.so/8008_postprocess_dns_whois_age.conf +/usr/share/logstash/pipeline.so/8200_postprocess_tagging.conf +#/usr/share/logstash/pipeline.so/8502_postprocess_freq_analysis_bro_dns.conf +#/usr/share/logstash/pipeline.so/8503_postprocess_freq_analysis_bro_http.conf +#/usr/share/logstash/pipeline.so/8504_postprocess_freq_analysis_bro_ssl.conf +#/usr/share/logstash/pipeline.so/8505_postprocess_freq_analysis_bro_x509.conf +/usr/share/logstash/pipeline.so/8998_postprocess_log_elapsed.conf +/usr/share/logstash/pipeline.so/8999_postprocess_rename_type.conf +/usr/share/logstash/pipeline.dynamic/9000_output_bro.conf +/usr/share/logstash/pipeline.dynamic/9001_output_switch.conf +/usr/share/logstash/pipeline.dynamic/9002_output_import.conf +/usr/share/logstash/pipeline.dynamic/9004_output_flow.conf +/usr/share/logstash/pipeline.dynamic/9026_output_dhcp.conf +/usr/share/logstash/pipeline.dynamic/9029_output_esxi.conf +/usr/share/logstash/pipeline.dynamic/9030_output_greensql.conf +/usr/share/logstash/pipeline.dynamic/9031_output_iis.conf +/usr/share/logstash/pipeline.dynamic/9032_output_mcafee.conf +/usr/share/logstash/pipeline.dynamic/9033_output_snort.conf +/usr/share/logstash/pipeline.dynamic/9034_output_syslog.conf +/usr/share/logstash/pipeline.dynamic/9200_output_firewall.conf +/usr/share/logstash/pipeline.dynamic/9300_output_windows.conf +/usr/share/logstash/pipeline.dynamic/9301_output_dns_windows.conf +/usr/share/logstash/pipeline.dynamic/9400_output_suricata.conf +/usr/share/logstash/pipeline.dynamic/9500_output_beats.conf +/usr/share/logstash/pipeline.dynamic/9600_output_ossec.conf +/usr/share/logstash/pipeline.dynamic/9998_output_test_data.conf diff --git a/salt/top.sls b/salt/top.sls index 0b2c50ca4..57844d57d 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -28,6 +28,7 @@ base: - suricata - bro - curator + - filebeat - utility - schedule From 3ffdcd81d66655443e6cef135e0e017f0cec7dbd Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 6 Dec 2018 03:07:16 +0000 Subject: [PATCH 034/110] Redis - Don't start if using Eval Mode --- salt/top.sls | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/top.sls b/salt/top.sls index 57844d57d..328d10a44 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -20,7 +20,6 @@ base: - firewall - master - idstools - - redis - elasticsearch - logstash - kibana From e7be61bd2883648eaebd14a6901d6cdcd3ade614 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 6 Dec 2018 03:13:39 +0000 Subject: [PATCH 035/110] Cleanup - Remove somaster module --- salt/somaster/init.sls | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100644 salt/somaster/init.sls diff --git a/salt/somaster/init.sls b/salt/somaster/init.sls deleted file mode 100644 index 3545ef846..000000000 --- a/salt/somaster/init.sls +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2014,2015,2016,2017,2018 Security Onion Solutions, LLC - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# Add Redis docker if REDIS is enabled -# Add REDIS user - -# Sync updated logstash config for REDIS - -# Add ES user - - -# Add ES Docker From 52dfe2500a9451c49ea9e94e020decf7f3517d5d Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 6 Dec 2018 03:14:03 +0000 Subject: [PATCH 036/110] Cleanup - Remove pulledpork module --- salt/pulledpork/etc/disablesid.conf | 38 ----- salt/pulledpork/etc/dropsid.conf | 42 ------ salt/pulledpork/etc/enablesid.conf | 48 ------- salt/pulledpork/etc/modifysid.conf | 40 ------ salt/pulledpork/etc/pulledpork.conf | 214 ---------------------------- salt/pulledpork/init.sls | 55 ------- 6 files changed, 437 deletions(-) delete mode 100644 salt/pulledpork/etc/disablesid.conf delete mode 100644 salt/pulledpork/etc/dropsid.conf delete mode 100644 salt/pulledpork/etc/enablesid.conf delete mode 100644 salt/pulledpork/etc/modifysid.conf delete mode 100644 salt/pulledpork/etc/pulledpork.conf delete mode 100644 salt/pulledpork/init.sls diff --git a/salt/pulledpork/etc/disablesid.conf b/salt/pulledpork/etc/disablesid.conf deleted file mode 100644 index 7e2381aa3..000000000 --- a/salt/pulledpork/etc/disablesid.conf +++ /dev/null @@ -1,38 +0,0 @@ -# example disablesid.conf V3.1 - -# Example of modifying state for individual rules -# 1:1034,1:9837,1:1270,1:3390,1:710,1:1249,3:13010 - -# Example of modifying state for rule ranges -# 1:220-1:3264,3:13010-3:13013 - -# Comments are allowed in this file, and can also be on the same line -# As the modify state syntax, as long as it is a trailing comment -# 1:1011 # I Disabled this rule because I could! - -# Example of modifying state for MS and cve rules, note the use of the : -# in cve. This will modify MS09-008, cve 2009-0233, bugtraq 21301, -# and all MS00 and all cve 2000 related sids! These support regular expression -# matching only after you have specified what you are looking for, i.e. -# MS00- or cve:, the first section CANNOT contain a regular -# expression (MS\d{2}-\d+) will NOT work, use the pcre: keyword (below) -# for this. -# MS09-008,cve:2009-0233,bugtraq:21301,MS00-\d+,cve:2000-\d+ - -# Example of using the pcre: keyword to modify rulestate. the pcre keyword -# allows for full use of regular expression syntax, you do not need to designate -# with / and all pcre searches are treated as case insensitive. For more information -# about regular expression syntax: http://www.regular-expressions.info/ -# The following example modifies state for all MS07 through MS10 -# pcre:MS(0[7-9]|10)-\d+ - -# Example of modifying state for specific categories entirely (see README.CATEGORIES) -# VRT-web-iis,ET-shellcode,ET-emergingthreats-smtp,Custom-shellcode,Custom-emergingthreats-smtp - -# Any of the above values can be on a single line or multiple lines, when -# on a single line they simply need to be separated by a , -# 1:9837,1:220-1:3264,3:13010-3:13013,pcre:MS(0[0-7])-\d+,MS09-008,cve:2009-0233 - -# The modifications in this file are for sample/example purposes only and -# should not actively be used, you need to modify this file to fit your -# environment. diff --git a/salt/pulledpork/etc/dropsid.conf b/salt/pulledpork/etc/dropsid.conf deleted file mode 100644 index 27a41e57e..000000000 --- a/salt/pulledpork/etc/dropsid.conf +++ /dev/null @@ -1,42 +0,0 @@ -# example dropsid.conf V3.1 -# -# Note: This file is used to specify what rules you wish to be set to have -# an action of drop rather than alert. This means that you are running -# snort inline (more info about inline deployments at snort.org). - -# Example of modifying state for individual rules -# 1:1034,1:9837,1:1270,1:3390,1:710,1:1249,3:13010 - -# Example of modifying state for rule ranges -# 1:220-1:3264,3:13010-3:13013 - -# Comments are allowed in this file, and can also be on the same line -# As the modify state syntax, as long as it is a trailing comment -# 1:1011 # I Disabled this rule because I could! - -# Example of modifying state for MS and cve rules, note the use of the : -# in cve. This will modify MS09-008, cve 2009-0233, bugtraq 21301, -# and all MS00 and all cve 2000 related sids! These support regular expression -# matching only after you have specified what you are looking for, i.e. -# MS00- or cve:, the first section CANNOT contain a regular -# expression (MS\d{2}-\d+) will NOT work, use the pcre: keyword (below) -# for this. -# MS09-008,cve:2009-0233,bugtraq:21301,MS00-\d+,cve:2000-\d+ - -# Example of using the pcre: keyword to modify rulestate. the pcre keyword -# allows for full use of regular expression syntax, you do not need to designate -# with / and all pcre searches are treated as case insensitive. For more information -# about regular expression syntax: http://www.regular-expressions.info/ -# The following example modifies state for all MS07 through MS10 -# pcre:MS(0[7-9]|10)-\d+ - -# Example of modifying state for specific categories entirely (see README.CATEGORIES) -# VRT-web-iis,ET-shellcode,ET-emergingthreats-smtp,Custom-shellcode,Custom-emergingthreats-smtp - -# Any of the above values can be on a single line or multiple lines, when -# on a single line they simply need to be separated by a , -# 1:9837,1:220-1:3264,3:13010-3:13013,pcre:MS(0[0-7])-\d+,MS09-008,cve:2009-0233 - -# The modifications in this file are for sample/example purposes only and -# should not actively be used, you need to modify this file to fit your -# environment. diff --git a/salt/pulledpork/etc/enablesid.conf b/salt/pulledpork/etc/enablesid.conf deleted file mode 100644 index 261f605e4..000000000 --- a/salt/pulledpork/etc/enablesid.conf +++ /dev/null @@ -1,48 +0,0 @@ -# example enablesid.conf v3.1 - -# SPECIAL NOTE, if you use the -R flag, the rule(s) specified in this file -# will be set back to their ORIGINAL state as it was read when they were -# originally extracted from the source tarball! - -# Example of modifying state for individual rules -# 1:1034,1:9837,1:1270,1:3390,1:710,1:1249,3:13010 - -# Example of modifying state for rule ranges -# 1:220-1:3264,3:13010-3:13013 - -# Comments are allowed in this file, and can also be on the same line -# As the modify state syntax, as long as it is a trailing comment -# 1:1011 # I Disabled this rule because I could! - -# Example of modifying state for MS and cve rules, note the use of the : -# in cve. This will modify MS09-008, cve 2009-0233, bugtraq 21301, -# and all MS00 and all cve 2000 related sids! These support regular expression -# matching only after you have specified what you are looking for, i.e. -# MS00- or cve:, the first section CANNOT contain a regular -# expression (MS\d{2}-\d+) will NOT work, use the pcre: keyword (below) -# for this. -# MS09-008,cve:2009-0233,bugtraq:21301,MS00-\d+,cve:2000-\d+ - -# Example of using the pcre: keyword to modify rulestate. the pcre keyword -# allows for full use of regular expression syntax, you do not need to designate -# with / and all pcre searches are treated as case insensitive. For more information -# about regular expression syntax: http://www.regular-expressions.info/ -# The following example modifies state for all MS07 through MS10 -# pcre:MS(0[7-9]|10)-\d+ - -# FOR TESTING ONLY: -# The following will enable ALL signatures for which Pulledpork has been configured -# to download -# pcre:. - -# Example of modifying state for specific categories entirely (see README.CATEGORIES) -# VRT-web-iis,ET-shellcode,ET-emergingthreats-smtp,Custom-shellcode,Custom-emergingthreats-smtp - -# Any of the above values can be on a single line or multiple lines, when -# on a single line they simply need to be separated by a , -# 1:9837,1:220-1:3264,3:13010-3:13013,pcre:MS(0[0-7])-\d+,MS09-008,cve:2009-0233 - -# The modifications in this file are for sample/example purposes only and -# should not actively be used, you need to modify this file to fit your -# environment. - diff --git a/salt/pulledpork/etc/modifysid.conf b/salt/pulledpork/etc/modifysid.conf deleted file mode 100644 index 50ee97601..000000000 --- a/salt/pulledpork/etc/modifysid.conf +++ /dev/null @@ -1,40 +0,0 @@ -# example modifysid.conf v1.1 2/18/2011 Alan Ptak -# -# Change history: -# ----------------------------------------------- -# v1.1 2/18/2011 Alan Ptak -# - Inserted comments around example elements that would otherwise modify rules -# -# v1.0 7/25/2010 JJC -# - original release -# ----------------------------------------------- -# -# formatting is simple -# "what I'm replacing" "what I'm replacing it with" -# -# Note that this will only work with GID:1 rules, simply because modifying -# GID:3 stub rules would not actually affect the rule, thusly it will remain -# non modifyable! -# -# If you are attempting to change rulestate (enable,drop,disable) from here -# then you are doing it wrong, it is much more efficient to do so from within -# the respective rulestate modification configuration files, please see doc/ -# and the README file! - -# the following applies to sid 10010 only and represents what would normally -# be s/to_client/from_server/ -# 10010 "to_client" "from_server" - -# the following would replace HTTP_PORTS with HTTPS_PORTS for ALL GID:1 -# rules -# "HTTP_PORTS" "HTTPS_PORTS" - -# multiple sids can be specified as noted below: -# 302,429,1821 "\$EXTERNAL_NET" "$HOME_NET" - -# example of modification of a rule to make snortsam BLOCK the rule: -# note that one rule changes from alert to BLOCK and that the other -# modifies the msg:" field value so that when the alert occurs it is noted -# that it is a SNORTSAM block rule! -# 17803 "\(msg:"" "\(msg:"SNORTSAM "; -# 17803 "^\s*alert" "BLOCK"; diff --git a/salt/pulledpork/etc/pulledpork.conf b/salt/pulledpork/etc/pulledpork.conf deleted file mode 100644 index daa5fcb17..000000000 --- a/salt/pulledpork/etc/pulledpork.conf +++ /dev/null @@ -1,214 +0,0 @@ -# Config file for pulledpork -# Be sure to read through the entire configuration file -# If you specify any of these items on the command line, it WILL take -# precedence over any value that you specify in this file! - -####### -####### The below section defines what your oinkcode is (required for -####### VRT rules), defines a temp path (must be writable) and also -####### defines what version of rules that you are getting (for your -####### snort version and subscription etc...) -####### - -# You can specify one or as many rule_urls as you like, they -# must appear as http://what.site.com/|rulesfile.tar.gz|1234567. You can specify -# each on an individual line, or you can specify them in a , separated list -# i.e. rule_url=http://x.y.z/|a.tar.gz|123,http://z.y.z/|b.tar.gz|456 -# note that the url, rule file, and oinkcode itself are separated by a pipe | -# i.e. url|tarball|123456789, -#rule_url=https://www.snort.org/reg-rules/|snortrules-snapshot.tar.gz| -# NEW Community ruleset: -#rule_url=https://snort.org/downloads/community/|community-rules.tar.gz|Community -# NEW For IP Blacklisting! Note the format is urltofile|IPBLACKLIST| -# This format MUST be followed to let pulledpork know that this is a blacklist -#rule_url=http://talosintelligence.com/feeds/ip-filter.blf|IPBLACKLIST|open -# URL for rule documentation! (slow to process) -#rule_url=https://snort.org/downloads/community/|opensource.tar.gz|Opensource -# THE FOLLOWING URL is for emergingthreats downloads, note the tarball name change! -# and open-nogpl, to avoid conflicts. -rule_url=https://rules.emergingthreats.net/open/suricata-4.0/|emerging.rules.tar.gz|open -# THE FOLLOWING URL is for etpro downloads, note the tarball name change! -# and the et oinkcode requirement! -#rule_url=https://rules.emergingthreatspro.com/|etpro.rules.tar.gz| -# NOTE above that the VRT snortrules-snapshot does not contain the version -# portion of the tarball name, this is because PP now automatically populates -# this value for you, if, however you put the version information in, PP will -# NOT populate this value but will use your value! - -# Specify rule categories to ignore from the tarball in a comma separated list -# with no spaces. There are four ways to do this: -# 1) Specify the category name with no suffix at all to ignore the category -# regardless of what rule-type it is, ie: netbios -# 2) Specify the category name with a '.rules' suffix to ignore only gid 1 -# rulefiles located in the /rules directory of the tarball, ie: policy.rules -# 3) Specify the category name with a '.preproc' suffix to ignore only -# preprocessor rules located in the /preproc_rules directory of the tarball, -# ie: sensitive-data.preproc -# 4) Specify the category name with a '.so' suffix to ignore only shared-object -# rules located in the /so_rules directory of the tarball, ie: netbios.so -# The example below ignores dos rules wherever they may appear, sensitive- -# data preprocessor rules, p2p so-rules (while including gid 1 p2p rules), -# and netbios gid-1 rules (while including netbios so-rules): -# ignore = dos,sensitive-data.preproc,p2p.so,netbios.rules -# These defaults are reasonable for the VRT ruleset with Snort 2.9.0.x. -ignore=deleted.rules,experimental.rules,local.rules -# IMPORTANT, if you are NOT yet using 2.8.6 then you MUST comment out the -# previous ignore line and uncomment the following! -# ignore=deleted,experimental,local,decoder,preprocessor,sensitive-data - -# What is our temp path, be sure this path has a bit of space for rule -# extraction and manipulation, no trailing slash -temp_path=/tmp - -####### -####### The below section is for rule processing. This section is -####### required if you are not specifying the configuration using -####### runtime switches. Note that runtime switches do SUPERSEED -####### any values that you have specified here! -####### - -# What path you want the .rules file containing all of the processed -# rules? (this value has changed as of 0.4.0, previously we copied -# all of the rules, now we are creating a single large rules file -# but still keeping a separate file for your so_rules! -rule_path=/opt/so/rules/nids/downloaded.rules - -# What path you want the .rules files to be written to, this is UNIQUE -# from the rule_path and cannot be used in conjunction, this is to be used with the -# -k runtime flag, this can be set at runtime using the -K flag or specified -# here. If specified here, the -k option must also be passed at runtime, however -# specifying -K at runtime forces the -k option to also be set -# out_path=/usr/local/etc/snort/rules/ - -# If you are running any rules in your local.rules file, we need to -# know about them to properly build a sid-msg.map that will contain your -# local.rules metadata (msg) information. You can specify other rules -# files that are local to your system here by adding a comma and more paths... -# remember that the FULL path must be specified for EACH value. -# local_rules=/path/to/these.rules,/path/to/those.rules -local_rules=/opt/so/rules/nids/local.rules,/opt/so/rules/nids/decoder-events.rules,/opt/so/rules/nids/stream-events.rules,/opt/so/rules/nids/http-events.rules,/opt/so/rules/nids/smtp-events.rules - -# Where should I put the sid-msg.map file? -sid_msg=/opt/so/rules/nids/sid-msg.map - -# New for by2 and more advanced msg mapping. Valid options are 1 or 2 -# specify version 2 if you are running barnyard2.2+. Otherwise use 1 -sid_msg_version=1 - -# Where do you want me to put the sid changelog? This is a changelog -# that pulledpork maintains of all new sids that are imported -sid_changelog=/var/log/nsm/sid_changes.log -# this value is optional - -####### -####### The below section is for so_rule processing only. If you don't -####### need to use them.. then comment this section out! -####### Alternately, if you are not using pulledpork to process -####### so_rules, you can specify -T at runtime to bypass this altogether -####### - -# What path you want the .so files to actually go to *i.e. where is it -# defined in your snort.conf, needs a trailing slash -sorule_path=/usr/local/lib/snort_dynamicrules/ - -# Path to the snort binary, we need this to generate the stub files -snort_path=/usr/bin/snort - -# We need to know where your snort.conf file lives so that we can -# generate the stub files -config_path=/etc/nsm/templates/snort/snort.conf - -##### Deprecated - The stubs are now categorically written to the single rule file! -# sostub_path=/usr/local/etc/snort/rules/so_rules.rules - -# Define your distro, this is for the precompiled shared object libs! -# Valid Distro Types: -# Debian-6-0, Ubuntu-10-4 -# Ubuntu-12-04, Centos-5-4 -# FC-12, FC-14, RHEL-5-5, RHEL-6-0 -# FreeBSD-8-1, FreeBSD-9-0, FreeBSD-10-0 -# OpenBSD-5-2, OpenBSD-5-3 -# OpenSUSE-11-4, OpenSUSE-12-1 -# Slackware-13-1 -distro=Centos-5-4 - -####### This next section is optional, but probably pretty useful to you. -####### Please read thoroughly! - -# If you are using IP Reputation and getting some public lists, you will probably -# want to tell pulledpork where your blacklist file lives, PP automagically will -# de-dupe any duplicate IPs from different sources. -black_list=/usr/local/etc/snort/rules/iplists/default.blacklist - -# IP Reputation does NOT require a full snort HUP, it introduces a concept whereby -# the IP list can be reloaded while snort is running through the use of a control -# socket. Please be sure that you built snort with the following optins: -# -enable-shared-rep and --enable-control-socket. Be sure to read about how to -# configure these! The following option tells pulledpork where to place the version -# file for use with control socket ip list reloads! -# This should be the same path where your black_list lives! -IPRVersion=/usr/local/etc/snort/rules/iplists - -# The following option tells snort where the snort_control tool is located. -snort_control=/usr/local/bin/snort_control - -# What do you want to backup and archive? This is a comma separated list -# of file or directory values. If a directory is specified, PP will recurse -# through said directory and all subdirectories to archive all files. -# The following example backs up all snort config files, rules, pulledpork -# config files, and snort shared object binary rules. -# backup=/usr/local/etc/snort,/usr/local/etc/pulledpork,/usr/local/lib/snort_dynamicrules/ - -# what path and filename should we use for the backup tarball? -# note that an epoch time value and the .tgz extension is automatically added -# to the backup_file name on completeion i.e. the written file is: -# pp_backup.1295886020.tgz -# backup_file=/tmp/pp_backup - -# Where do you want the signature docs to be copied, if this is commented -# out then they will not be copied / extracted. Note that extracting them -# will add considerable runtime to pulledpork. -# docs=/path/to/base/www - -# The following option, state_order, allows you to more finely control the order -# that pulledpork performs the modify operations, specifically the enablesid -# disablesid and dropsid functions. An example use case here would be to -# disable an entire category and later enable only a rule or two out of it. -# the valid values are disable, drop, and enable. -# state_order=disable,drop,enable - - -# Define the path to the pid files of any running process that you want to -# HUP after PP has completed its run. -# pid_path=/var/run/snort.pid,/var/run/barnyard.pid,/var/run/barnyard2.pid -# and so on... -# pid_path=/var/run/snort_eth0.pid - -# This defines the version of snort that you are using, for use ONLY if the -# proper snort binary is not on the system that you are fetching the rules with -# This value MUST contain all 4 minor version -# numbers. ET rules are now also dependant on this, verify supported ET versions -# prior to simply throwing rubbish in this variable kthx! -# -# Suricata users - set this to 'suricata-3.x.x' to process rule files -# for suricata, this mimics the -S flag on the command line. -# snort_version=2.9.0.0 - -# Here you can specify what rule modification files to run automatically. -# simply uncomment and specify the apt path. -enablesid=/opt/so/pulledpork/etc/enablesid.conf -dropsid=/opt/so/pulledpork/dropsid.conf -disablesid=/opt/so/pulledpork/disablesid.conf -modifysid=/opt/so/pulledpork/modifysid.conf - -# What is the base ruleset that you want to use, please uncomment to use -# and see the README.RULESETS for a description of the options. -# Note that setting this value will disable all ET rulesets if you are -# Running such rulesets -# ips_policy=security - -####### Remember, a number of these values are optional.. if you don't -####### need to process so_rules, simply comment out the so_rule section -####### you can also specify -T at runtime to process only GID 1 rules. - -version=0.7.3 diff --git a/salt/pulledpork/init.sls b/salt/pulledpork/init.sls deleted file mode 100644 index e1ae2728f..000000000 --- a/salt/pulledpork/init.sls +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2014,2015,2016,2017,2018 Security Onion Solutions, LLC - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# PulledProk Setup -ppdir: - file.directory: - - name: /opt/so/pulledpork/etc - - user: 939 - - group: 939 - - makedirs: True - -ppetcsync: - file.recurse: - - name: /opt/so/pulledpork/etc - - source: salt://pulledpork/etc - - user: 939 - - group: 939 - - template: jinja - -rulesdir: - file.directory: - - name: /opt/so/rules/nids - - user: 939 - - group: 939 - - makedirs: True - -ruleslink: - file.symlink: - - name: /opt/so/saltstack/salt/pulledpork/rules - - target: /opt/so/rules/nids - -toosmooth/so-pulledpork:test2: - docker_image.present - -so-pulledpork: - docker_container.running: - - image: toosmooth/so-pulledpork:test2 - - hostname: so-pulledpork - - user: socore - - binds: - - /opt/so/pulledpork/etc:/opt/pulledpork/etc:ro - - /opt/so/rules/nids:/opt/so/rules/nids:rw - - network_mode: so-elastic-net From fe1c771f211991201e87f82e9158d15cb835c10b Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 6 Dec 2018 05:42:30 +0000 Subject: [PATCH 037/110] Elastalert - Initial Config --- salt/elastalert/init.sls | 101 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 salt/elastalert/init.sls diff --git a/salt/elastalert/init.sls b/salt/elastalert/init.sls new file mode 100644 index 000000000..28722fe01 --- /dev/null +++ b/salt/elastalert/init.sls @@ -0,0 +1,101 @@ +# Copyright 2014,2015,2016,2017,2018 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +{% if grains['role'] == 'so-master' %} + +{% set esalert = salt['pillar.get']('master:elastalert', '1') %} +{% set esip = salt['pillar.get']('master:mainip', '') %} +{% set esport = salt['pillar.get']('master:es_port', '') %} + + +{% elif grains['role'] == 'so-eval' %} + +{% set esalert = salt['pillar.get']('master:elastalert', '1') %} +{% set esip = salt['pillar.get']('master:mainip', '') %} +{% set esport = salt['pillar.get']('master:es_port', '') %} + + +{% elif grains['role'] == 'so-node' %} + +{% set esalert = salt['pillar.get']('node:elastalert', '0') %} + +{% endif %} + +# Elastalert +{% if esalert == 1 %} + +# Create the group +elastagroup: + group.present: + - name: elastalert + - gid: 933 + +# Add user +elastalert: + user.present: + - uid: 933 + - gid: 933 + - home: /opt/so/conf/elastalert + - createhome: False + +elastalogdir: + file.directory: + - name: /opt/so/log/elastalert + - user: 933 + - group: 939 + - makedirs: True + +elastarules: + file.directory: + - name: /opt/so/rules/elastalert + - user: 933 + - group: 939 + - makedirs: True + +#elastaconfdir: +# file.directory: +# - name: /opt/so/conf/elastalert +# - user: 933 +# - group: 939 +# - makedirs: True + +#elastaconf: +# file.managed: +# - name: /opt/so/conf/elastalert/config.yaml +# - source: salt://elastalert/files/config.yaml +# - user: 933 +# - group: 939 +# - template: jinja + +so-elastalert: + docker_container.running: + - image: soshybridhunter/so-elastalert:HH1.0.3 + - hostname: elastalert + - name: so-elastalert + - user: elastalert + - detach: True + - binds: +# - /opt/so/conf/elastalert/config.yaml:/etc/elastalert/conf/elastalert_config.yaml:ro + - /opt/so/rules/elastalert:/etc/elastalert/rules/:ro + - /opt/so/log/elastalert:/var/log/elastalert:rw + - environment: + - ELASTICSEARCH_HOST: {{ esip }} + - ELASTICSEARCH_PORT: {{ esport }} + - ELASTALERT_CONFIG: /etc/elastalert/conf/elastalert_config.yaml + - ELASTALERT_SUPERVISOR_CONF: /etc/elastalert/conf/elastalert_supervisord.conf + - RULES_DIRECTORY: /etc/elastalert/rules/ + - LOG_DIR: /var/log/elastalert + +{% endif %} From d310f163e80024bec6cb07770377273cb6b12da3 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 6 Dec 2018 05:43:15 +0000 Subject: [PATCH 038/110] Elastalert - Enable for Master/Eval --- salt/top.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/top.sls b/salt/top.sls index 328d10a44..7381af727 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -27,6 +27,7 @@ base: - suricata - bro - curator + - elastalert - filebeat - utility - schedule @@ -43,6 +44,7 @@ base: - elasticsearch - logstash - kibana + - elastalert - utility - schedule From e335be747651c6202930245f30a467207761faee Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 6 Dec 2018 05:44:09 +0000 Subject: [PATCH 039/110] Elastalert - Remove config from ES Module --- salt/elasticsearch/init.sls | 54 ------------------------------------- 1 file changed, 54 deletions(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 122e2f7eb..27bc883f4 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -18,7 +18,6 @@ {% set esheap = salt['pillar.get']('master:esheap', '') %} {% set freq = salt['pillar.get']('master:freq', '0') %} {% set dstats = salt['pillar.get']('master:dstats', '0') %} -{% set esalert = salt['pillar.get']('master:elastalert', '1') %} {% elif grains['role'] == 'so-eval' %} @@ -26,7 +25,6 @@ {% set esheap = salt['pillar.get']('master:esheap', '') %} {% set freq = salt['pillar.get']('master:freq', '0') %} {% set dstats = salt['pillar.get']('master:dstats', '0') %} -{% set esalert = salt['pillar.get']('master:elastalert', '1') %} {% elif grains['role'] == 'so-node' %} @@ -34,7 +32,6 @@ {% set esheap = salt['pillar.get']('node:esheap', '') %} {% set freq = salt['pillar.get']('node:freq', '0') %} {% set dstats = salt['pillar.get']('node:dstats', '0') %} -{% set esalert = salt['pillar.get']('node:elastalert', '1') %} {% endif %} @@ -193,54 +190,3 @@ so-domainstats: {% endif %} - -# Elastalert -{% if esalert == 1 %} - -# Create the group -elastagroup: - group.present: - - name: elastalert - - gid: 933 - -# Add user -elastalert: - user.present: - - uid: 933 - - gid: 933 - - home: /opt/so/conf/elastalert - - createhome: False - -elastalogdir: - file.directory: - - name: /opt/so/log/elastalert - - user: 933 - - group: 939 - - makedirs: True - -elastarules: - file.directory: - - name: /opt/so/rules/elastalert - - user: 933 - - group: 939 - - makedirs: True - -elastaconf: - file.directory: - - name: /opt/so/conf/elastalert - - user: 933 - - group: 939 - - makedirs: True - -so-elastalert: - docker_container.running: - - image: soshybridhunter/so-elastalert:HH1.0.3 - - hostname: elastalert - - name: so-elastalert - - user: elastalert - - detach: True - - binds: - - /etc/elastalert/rules/:/etc/elastalert/rules/:ro - - /opt/so/log/elastalert:/var/log/elastalert:rw - -{% endif %} From 46ef0280cd3e857a1f86b369c1b0961280f5b034 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 6 Dec 2018 11:11:25 -0500 Subject: [PATCH 040/110] Fleet Module - Move database stuff to the fleet module --- salt/fleet/init.sls | 18 ++++++++++++++++++ salt/mysql/init.sls | 18 ------------------ salt/top.sls | 6 +++++- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/salt/fleet/init.sls b/salt/fleet/init.sls index 59c02c0ce..381ce914b 100644 --- a/salt/fleet/init.sls +++ b/salt/fleet/init.sls @@ -30,3 +30,21 @@ so-fleet: - /etc/pki/fleet.crt:/ssl/server.cert - watch: - /opt/so/conf/fleet/etc + +fleetdb: + mysql_database.present: + - name: fleet + +fleetdbuser: + mysql_user.present: + - host: 172.17.0.0/255.255.0.0 + - password: {{ FLEETPASS }} + - connection_user: root + - connection_pass: {{ MYSQLPASS }} + +fleetdbpriv: + mysql_grants.present: + - grant: all privileges + - database: fleet.* + - user: fleetdbuser + - host: 172.17.0.0/255.255.0.0 diff --git a/salt/mysql/init.sls b/salt/mysql/init.sls index 36faf9675..0996ed3a7 100644 --- a/salt/mysql/init.sls +++ b/salt/mysql/init.sls @@ -65,21 +65,3 @@ so-mysql: - /opt/so/log/mysql:/var/log/mysql:rw - watch: - /opt/so/conf/mysql/etc - -fleetdb: - mysql_database.present: - - name: fleet - -fleetdbuser: - mysql_user.present: - - host: 172.17.0.0/255.255.0.0 - - password: {{ FLEETPASS }} - - connection_user: root - - connection_pass: {{ MYSQLPASS }} - -fleetdbpriv: - mysql_grants.present: - - grant: all privileges - - database: fleet.* - - user: fleetdbuser - - host: 172.17.0.0/255.255.0.0 diff --git a/salt/top.sls b/salt/top.sls index 0b2c50ca4..7db5d45b8 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -21,6 +21,7 @@ base: - master - idstools - redis + - mysql - elasticsearch - logstash - kibana @@ -29,7 +30,8 @@ base: - bro - curator - utility - - schedule + - schedule + - fleet 'G@role:so-master': @@ -40,11 +42,13 @@ base: - master - idstools - redis + - mysql - elasticsearch - logstash - kibana - utility - schedule + - fleet # Storage node logic From 43d75ba07d821015ccee7046054a4ab061d3d447 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 6 Dec 2018 11:17:04 -0500 Subject: [PATCH 041/110] Fleet Module - Move database stuff to the fleet module --- salt/top.sls | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/salt/top.sls b/salt/top.sls index fcd9ba98d..05e027a84 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -20,11 +20,7 @@ base: - firewall - master - idstools -<<<<<<< HEAD - - redis - mysql -======= ->>>>>>> 7816c6e18e21870b0249cc05a161ef1644a94066 - elasticsearch - logstash - kibana @@ -36,7 +32,7 @@ base: - filebeat - utility - schedule - - fleet + - fleet 'G@role:so-master': From 4e28feb084af1ae402ed7a8a03389808f7b501cd Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 6 Dec 2018 11:18:13 -0500 Subject: [PATCH 042/110] Mysql Module - fix nameing --- salt/mysql/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/mysql/init.sls b/salt/mysql/init.sls index 0996ed3a7..785e2702a 100644 --- a/salt/mysql/init.sls +++ b/salt/mysql/init.sls @@ -26,7 +26,7 @@ mysqlpiddir: - group: 939 - makedirs: True -lsetcsync: +mysqletcsync: file.recurse: - name: /opt/so/conf/mysql/etc - source: salt://mysql/etc From bcd7773aceed9cf099869afe2de322b9a0170d87 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 6 Dec 2018 11:19:44 -0500 Subject: [PATCH 043/110] Fleet Module - fix assignment of MYSQLPASS --- salt/fleet/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/fleet/init.sls b/salt/fleet/init.sls index 381ce914b..9ecd6b661 100644 --- a/salt/fleet/init.sls +++ b/salt/fleet/init.sls @@ -1,3 +1,4 @@ +{%- set MYSQLPASS = salt['pillar.get']('master:mysqlpass', 'iwonttellyou') %} {%- set FLEETPASS = salt['pillar.get']('master:fleetpass', 'bazinga') -%} {%- set MASTERIP = salt['pillar.get']('static:masterip', '') -%} From ac3eaaf1143cbde470d00e3b88e70ad6fa099d1d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 6 Dec 2018 13:51:59 -0500 Subject: [PATCH 044/110] Setup Script - Add MYSQL and fleet password generation --- so-setup-network.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index 13182a6f7..2ca925137 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -394,6 +394,13 @@ filter_nics() { FNICS=$(ip link | grep -vw $MNIC | awk -F: '$0 !~ "lo|vir|veth|br|docker|wl|^[^0-9]"{print $2 " \"" "Interface" "\"" " OFF"}') } + +generate_passwords(){ + # Generate Random Passwords for Things + MYSQLPASS=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 20 | head -n 1) + FLEETPASS=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 20 | head -n 1) +} + get_filesystem_nsm(){ FSNSM=$(df /nsm | awk '$3 ~ /[0-9]+/ { print $2 * 1000 }') } @@ -507,6 +514,8 @@ master_pillar() { echo " es_port: $NODE_ES_PORT" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls echo " log_size_limit: $LOG_SIZE_LIMIT" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls echo " cur_close_days: $CURCLOSEDAYS" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls + echo " mysqlpass: $MYSQLPASS" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls + echo " fleetpass: $FLEETPASS" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls } @@ -1395,6 +1404,7 @@ if (whiptail_you_sure); then # Last Chance to back out whiptail_make_changes + generate_passwords clear_master mkdir -p /nsm get_filesystem_root @@ -1549,6 +1559,7 @@ if (whiptail_you_sure); then BROVERSION=ZEEK CURCLOSEDAYS=30 whiptail_make_changes + generate_passwords clear_master mkdir -p /nsm get_filesystem_root @@ -1597,7 +1608,7 @@ if (whiptail_you_sure); then get_log_size_limit whiptail_log_size_limit CURCLOSEDAYS=30 - whiptail_cur_close_days + whiptail_cur_close_days es_heapsize ls_heapsize whiptail_node_advanced From 8315f9279e7bbbb9a498e52d680f72bf696718ff Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 6 Dec 2018 14:10:28 -0500 Subject: [PATCH 045/110] Fleet Module - Fix order --- salt/fleet/init.sls | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/salt/fleet/init.sls b/salt/fleet/init.sls index 9ecd6b661..4860ed816 100644 --- a/salt/fleet/init.sls +++ b/salt/fleet/init.sls @@ -10,6 +10,24 @@ fleetcdir: - group: 939 - makedirs: True +fleetdb: + mysql_database.present: + - name: fleet + +fleetdbuser: + mysql_user.present: + - host: 172.17.0.0/255.255.0.0 + - password: {{ FLEETPASS }} + - connection_user: root + - connection_pass: {{ MYSQLPASS }} + +fleetdbpriv: + mysql_grants.present: + - grant: all privileges + - database: fleet.* + - user: fleetdbuser + - host: 172.17.0.0/255.255.0.0 + so-fleet: docker_container.running: - image: kolide/fleet @@ -31,21 +49,3 @@ so-fleet: - /etc/pki/fleet.crt:/ssl/server.cert - watch: - /opt/so/conf/fleet/etc - -fleetdb: - mysql_database.present: - - name: fleet - -fleetdbuser: - mysql_user.present: - - host: 172.17.0.0/255.255.0.0 - - password: {{ FLEETPASS }} - - connection_user: root - - connection_pass: {{ MYSQLPASS }} - -fleetdbpriv: - mysql_grants.present: - - grant: all privileges - - database: fleet.* - - user: fleetdbuser - - host: 172.17.0.0/255.255.0.0 From 7d7cd313fa945587c45e1db75697aed811bd886b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 6 Dec 2018 14:25:31 -0500 Subject: [PATCH 046/110] Fleet Module - Use the SO docker --- salt/fleet/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/fleet/init.sls b/salt/fleet/init.sls index 4860ed816..7c1f0099d 100644 --- a/salt/fleet/init.sls +++ b/salt/fleet/init.sls @@ -30,7 +30,7 @@ fleetdbpriv: so-fleet: docker_container.running: - - image: kolide/fleet + - image: soshybridhunter/so-fleet:HH1.0.5 - hostname: so-fleet - port_bindings: - 0.0.0.0:8080:8080 From f5d16181d4bc5c6262c1cf92482ac37adb21ab05 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 6 Dec 2018 14:40:44 -0500 Subject: [PATCH 047/110] Common Module - Add Fleet to nginx --- salt/common/nginx/nginx.conf.so-eval | 11 +++++++++++ salt/common/nginx/nginx.conf.so-master | 13 +++++++++++++ 2 files changed, 24 insertions(+) diff --git a/salt/common/nginx/nginx.conf.so-eval b/salt/common/nginx/nginx.conf.so-eval index 07813d1a1..4ffb2835f 100644 --- a/salt/common/nginx/nginx.conf.so-eval +++ b/salt/common/nginx/nginx.conf.so-eval @@ -109,6 +109,17 @@ http { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; + } + location /fleet/ { + rewrite /fleet/(.*) /$1 break; + proxy_pass https://{{ masterip }}:8080/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + } error_page 404 /404.html; diff --git a/salt/common/nginx/nginx.conf.so-master b/salt/common/nginx/nginx.conf.so-master index 07813d1a1..535009c71 100644 --- a/salt/common/nginx/nginx.conf.so-master +++ b/salt/common/nginx/nginx.conf.so-master @@ -111,6 +111,19 @@ http { } + + location /fleet/ { + rewrite /fleet/(.*) /$1 break; + proxy_pass https://{{ masterip }}:8080/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } + error_page 404 /404.html; location = /40x.html { } From bccb83fae92447f16649dbaf1a3886cc900645ff Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 7 Dec 2018 03:57:31 +0000 Subject: [PATCH 048/110] Setup - Install Wazuh agent --- so-setup-network.sh | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/so-setup-network.sh b/so-setup-network.sh index 2ca925137..9c6eeda09 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -75,6 +75,22 @@ add_socore_user_notmaster() { } +add_wazuh_users() { + + if [ $OS == 'centos' ]; then + local ADDUSER=adduser + else + local ADDUSER=useradd + fi + + groupadd --gid 945 ossec + $ADDUSER --uid 943 --gid 945 --home-dir /opt/so/wazuh --no-create-home ossecm + $ADDUSER --uid 944 --gid 945 --home-dir /opt/so/wazuh --no-create-home ossecr + $ADDUSER --uid 945 --gid 945 --home-dir /opt/so/wazuh --no-create-home ossec + +} + + # Enable Bro Logs bro_logs_enabled() { @@ -875,6 +891,22 @@ update_sudoers() { } +wazuh_agent_install() { + + # Get key + curl -s https://packages.wazuh.com/key/GPG-KEY-WAZUH | apt-key add - + # Add repo + echo "deb https://packages.wazuh.com/3.x/apt/ stable main" | tee /etc/apt/sources.list.d/wazuh.list + apt-get update -y + # Install + apt-get install -y wazuh-agent + # Prevent automatic updates + sed -i "s/^deb/#deb/" /etc/apt/sources.list.d/wazuh.list + # Set package state to "hold" + echo "wazuh-agent hold" | sudo dpkg --set-selections + +} + ########################################### ## ## ## Whiptail Menu Section ## @@ -1421,6 +1453,12 @@ if (whiptail_you_sure); then echo "" add_socore_user_master + echo "** Adding Wazuh users **" + add_wazuh_users + + echo "** Installing Wazuh agent **" + wazuh_agent_install + # Install salt and dependencies echo " ** Installing Salt and Dependencies **" saltify >>~/sosetup.log 2>&1 @@ -1507,6 +1545,8 @@ if (whiptail_you_sure); then mkdir -p /nsm get_filesystem_root get_filesystem_nsm + add_wazuh_users + wazuh_agent_install copy_ssh_key set_initial_firewall_policy create_bond @@ -1571,6 +1611,8 @@ if (whiptail_you_sure); then echo "**** Please set a password for socore. You will use this password when setting up other Nodes/Sensors" echo "" add_socore_user_master + add_wazuh_users + wazuh_agent_instal create_bond saltify docker_install @@ -1632,6 +1674,8 @@ if (whiptail_you_sure); then mkdir -p /nsm get_filesystem_root get_filesystem_nsm + add_wazuh_users + wazuh_agent_install copy_ssh_key set_initial_firewall_policy saltify From e21c67f553448ead57747371cf03694d09571157 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 7 Dec 2018 04:14:30 +0000 Subject: [PATCH 049/110] Setup - Fix typo --- so-setup-network.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index 9c6eeda09..dc470487b 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -1612,7 +1612,7 @@ if (whiptail_you_sure); then echo "" add_socore_user_master add_wazuh_users - wazuh_agent_instal + wazuh_agent_install create_bond saltify docker_install From 6bfb813cdfd0e36702097d7d4c1c2c4a3c4d113f Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 7 Dec 2018 04:18:28 +0000 Subject: [PATCH 050/110] Setup - Adjust syntax --- so-setup-network.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index dc470487b..21b6e85e6 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -171,7 +171,7 @@ chown_salt_master() { clear_master() { # Clear out the old master public key in case this is a re-install. # This only happens if you re-install the master. - if [ -f /etc/salt/pki/minion/minion_master.pub]; then + if [ -f /etc/salt/pki/minion/minion_master.pub ]; then rm /etc/salt/pki/minion/minion_master.pub service salt-minion restart fi From f5e2b7f210b2a1be1e4b76b7d2c18411be15e445 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 7 Dec 2018 04:22:11 +0000 Subject: [PATCH 051/110] Setup - Adjust to prevent unary operator error --- so-setup-network.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index 21b6e85e6..63a214dc4 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -547,7 +547,7 @@ master_static() { echo " broversion: $BROVERSION" >> /opt/so/saltstack/pillar/static.sls echo " ids: $NIDS" >> /opt/so/saltstack/pillar/static.sls echo " masterip: $MAINIP" >> /opt/so/saltstack/pillar/static.sls - if [ $MASTERUPDATES == 'MASTER' ]; then + if [[ $MASTERUPDATES == 'MASTER' ]]; then echo " masterupdate: 1" >> /opt/so/saltstack/pillar/static.sls else echo " masterupdate: 0" >> /opt/so/saltstack/pillar/static.sls From e11aadf7306a1fdad74bfd2babbd39caa3c617a4 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 7 Dec 2018 13:36:11 +0000 Subject: [PATCH 052/110] Wazuh - agent install for CentOS --- salt/wazuh/files/filebeat.yml | 328 ++++++++++++++++++++++++++++++++-- so-setup-network.sh | 39 ++-- 2 files changed, 341 insertions(+), 26 deletions(-) diff --git a/salt/wazuh/files/filebeat.yml b/salt/wazuh/files/filebeat.yml index 7f076793c..7c6e0655b 100644 --- a/salt/wazuh/files/filebeat.yml +++ b/salt/wazuh/files/filebeat.yml @@ -1,16 +1,314 @@ -filebeat: - prospectors: - - input_type: log - paths: - - "/var/ossec/data/logs/alerts/alerts.json" - document_type: wazuh-alerts - json.message_key: log - json.keys_under_root: true - json.overwrite_keys: true +{%- set MASTER = grains['master'] %} +{%- set HOSTNAME = salt['grains.get']('host', '') %} +{%- set BROVER = salt['pillar.get']('static:broversion', 'COMMUNITY') %} -output: - logstash: - # The Logstash hosts - hosts: ["logstash:5000"] -# ssl: -# certificate_authorities: ["/etc/filebeat/logstash.crt"] +name: {{ HOSTNAME }} + +#========================== Modules configuration ============================ +filebeat.modules: +#=========================== Filebeat prospectors ============================= + +# List of prospectors to fetch data. +filebeat.prospectors: +#------------------------------ Log prospector -------------------------------- +{%- if BROVER != 'SURICATA' %} +{%- for LOGNAME in salt['pillar.get']('brologs:enabled', '') %} + - type: log + paths: + - /nsm/bro/logs/current/{{ LOGNAME }}.log + fields: + type: bro_{{ LOGNAME }} + fields_under_root: true + tags: ["bro"] + clean_removed: false + close_removed: false + +{%- endfor %} +{%- endif %} + + - type: log + paths: + - /suricata/eve.json + fields: + type: ids + engine: suricata + fields_under_root: true + clean_removed: false + close_removed: false + + - type: log + paths: + - /alerts/alerts.json + fields: + type: ossec + fields_under_root: true + clean_removed: false + close_removed: false + + + +#----------------------------- Logstash output --------------------------------- +output.logstash: + # Boolean flag to enable or disable the output module. + enabled: true + + # The Logstash hosts + hosts: ["{{ MASTER }}:5044"] + + # Number of workers per Logstash host. + worker: 1 + + # Set gzip compression level. + compression_level: 3 + + + # Enable SSL support. SSL is automatically enabled, if any SSL setting is set. + ssl.enabled: true + + # Configure SSL verification mode. If `none` is configured, all server hosts + # and certificates will be accepted. In this mode, SSL based connections are + # susceptible to man-in-the-middle attacks. Use only for testing. Default is + # `full`. + ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions 1.0 up to + # 1.2 are enabled. + #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] + + # Optional SSL configuration options. SSL is off by default. + # List of root certificates for HTTPS server verifications + ssl.certificate_authorities: ["/usr/share/filebeat/intraca.crt"] + + # Certificate for SSL client authentication + ssl.certificate: "/usr/share/filebeat/filebeat.crt" + + # Client Certificate Key + ssl.key: "/usr/share/filebeat/filebeat.key" + + +# Elasticsearch template settings +#setup.template.settings: + + # A dictionary of settings to place into the settings.index dictionary + # of the Elasticsearch template. For more details, please check + # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html + #index: + #number_of_shards: 1 + #codec: best_compression + #number_of_routing_shards: 30 + + # A dictionary of settings for the _source field. For more details, please check + # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html + #_source: + #enabled: false + +#============================== Kibana ===================================== + +# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. +# This requires a Kibana endpoint configuration. +#setup.kibana: + + # Kibana Host + # Scheme and port can be left out and will be set to the default (http and 5601) + # In case you specify and additional path, the scheme is required: http://localhost:5601/path + # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 + #host: "localhost:5601" + + # Optional protocol and basic auth credentials. + #protocol: "https" + #username: "elastic" + #password: "changeme" + + # Optional HTTP Path + #path: "" + + # Use SSL settings for HTTPS. Default is true. + #ssl.enabled: true + + # Configure SSL verification mode. If `none` is configured, all server hosts + # and certificates will be accepted. In this mode, SSL based connections are + # susceptible to man-in-the-middle attacks. Use only for testing. Default is + # `full`. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions 1.0 up to + # 1.2 are enabled. + #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] + + # SSL configuration. By default is off. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the Certificate Key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites + #ssl.curve_types: [] + + + +#================================ Logging ====================================== +# There are four options for the log output: file, stderr, syslog, eventlog +# The file output is the default. + +# Sets log level. The default log level is info. +# Available log levels are: error, warning, info, debug +logging.level: debug + +# Enable debug output for selected components. To enable all selectors use ["*"] +# Other available selectors are "beat", "publish", "service" +# Multiple selectors can be chained. +#logging.selectors: [ ] + +# Send all logging output to syslog. The default is false. +#logging.to_syslog: false + +# Send all logging output to Windows Event Logs. The default is false. +#logging.to_eventlog: false + +# If enabled, filebeat periodically logs its internal metrics that have changed +# in the last period. For each metric that changed, the delta from the value at +# the beginning of the period is logged. Also, the total values for +# all non-zero internal metrics are logged on shutdown. The default is true. +#logging.metrics.enabled: true + +# The period after which to log the internal metrics. The default is 30s. +#logging.metrics.period: 30s + +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +logging.to_files: true +logging.files: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + path: /var/log/filebeat + + # The name of the files where the logs are written to. + name: filebeat + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + keepfiles: 7 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + +# Set to true to log messages in json format. +#logging.json: false + + +#============================== Xpack Monitoring ===================================== +# filebeat can export internal metrics to a central Elasticsearch monitoring cluster. +# This requires xpack monitoring to be enabled in Elasticsearch. +# The reporting is disabled by default. + +# Set to true to enable the monitoring reporter. +#xpack.monitoring.enabled: false + +# Uncomment to send the metrics to Elasticsearch. Most settings from the +# Elasticsearch output are accepted here as well. Any setting that is not set is +# automatically inherited from the Elasticsearch output configuration, so if you +# have the Elasticsearch output configured, you can simply uncomment the +# following line, and leave the rest commented out. +#xpack.monitoring.elasticsearch: + + # Array of hosts to connect to. + # Scheme and port can be left out and will be set to the default (http and 9200) + # In case you specify and additional path, the scheme is required: http://localhost:9200/path + # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 + #hosts: ["localhost:9200"] + + # Set gzip compression level. + #compression_level: 0 + + # Optional protocol and basic auth credentials. + #protocol: "https" + #username: "beats_system" + #password: "changeme" + + # Dictionary of HTTP parameters to pass within the url with index operations. + #parameters: + #param1: value1 + #param2: value2 + + # Custom HTTP headers to add to each request + #headers: + # X-My-Header: Contents of the header + + # Proxy server url + #proxy_url: http://proxy:3128 + + # The number of times a particular Elasticsearch index operation is attempted. If + # the indexing operation doesn't succeed after this many retries, the events are + # dropped. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Elasticsearch bulk API index request. + # The default is 50. + #bulk_max_size: 50 + + # Configure http request timeout before failing an request to Elasticsearch. + #timeout: 90 + + # Use SSL settings for HTTPS. + #ssl.enabled: true + + # Configure SSL verification mode. If `none` is configured, all server hosts + # and certificates will be accepted. In this mode, SSL based connections are + # susceptible to man-in-the-middle attacks. Use only for testing. Default is + # `full`. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions 1.0 up to + # 1.2 are enabled. + #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] + + # SSL configuration. By default is off. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the Certificate Key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites + #ssl.curve_types: [] + + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + +#================================ HTTP Endpoint ====================================== +# Each beat can expose internal metrics through a HTTP endpoint. For security +# reasons the endpoint is disabled by default. This feature is currently experimental. +# Stats can be access through http://localhost:5066/stats . For pretty JSON output +# append ?pretty to the URL. + +# Defines if the HTTP endpoint is enabled. +#http.enabled: false + +# The HTTP endpoint will bind to this hostname or IP address. It is recommended to use only localhost. +#http.host: localhost + +# Port on which the HTTP endpoint will bind. Default is 5066. diff --git a/so-setup-network.sh b/so-setup-network.sh index 63a214dc4..240172838 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -893,17 +893,34 @@ update_sudoers() { wazuh_agent_install() { - # Get key - curl -s https://packages.wazuh.com/key/GPG-KEY-WAZUH | apt-key add - - # Add repo - echo "deb https://packages.wazuh.com/3.x/apt/ stable main" | tee /etc/apt/sources.list.d/wazuh.list - apt-get update -y - # Install - apt-get install -y wazuh-agent - # Prevent automatic updates - sed -i "s/^deb/#deb/" /etc/apt/sources.list.d/wazuh.list - # Set package state to "hold" - echo "wazuh-agent hold" | sudo dpkg --set-selections + if [ $OS == 'centos' ]; then + # Add repo + cat > /etc/yum.repos.d/wazuh.repo <<\EOF +[wazuh_repo] +gpgcheck=1 +gpgkey=https://packages.wazuh.com/key/GPG-KEY-WAZUH +enabled=1 +name=Wazuh repository +baseurl=https://packages.wazuh.com/3.x/yum/ +protect=1 +EOF + # Install agent + yum install -y wazuh-agent + # Prevent automatic upates + sed -i "s/^enabled=1/enabled=0/" /etc/yum.repos.d/wazuh.repo + else + # Get key + curl -s https://packages.wazuh.com/key/GPG-KEY-WAZUH | apt-key add - + # Add repo + echo "deb https://packages.wazuh.com/3.x/apt/ stable main" | tee /etc/apt/sources.list.d/wazuh.list + apt-get update -y + # Install + apt-get install -y wazuh-agent + # Prevent automatic updates + sed -i "s/^deb/#deb/" /etc/apt/sources.list.d/wazuh.list + # Set package state to "hold" + echo "wazuh-agent hold" | sudo dpkg --set-selections + fi } From e355503324e61f5dfaea7277a1f84da85bc8319a Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 7 Dec 2018 13:38:31 +0000 Subject: [PATCH 053/110] Filebeat - Update for Wazuh logs --- salt/filebeat/etc/filebeat.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index f8fb5acf5..7c6e0655b 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -36,6 +36,16 @@ filebeat.prospectors: clean_removed: false close_removed: false + - type: log + paths: + - /alerts/alerts.json + fields: + type: ossec + fields_under_root: true + clean_removed: false + close_removed: false + + #----------------------------- Logstash output --------------------------------- output.logstash: @@ -152,7 +162,7 @@ output.logstash: # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug -#logging.level: info +logging.level: debug # Enable debug output for selected components. To enable all selectors use ["*"] # Other available selectors are "beat", "publish", "service" From 6a9a537cf8d1e8ab6dc18fa7b7c1a0d773ae3d86 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 7 Dec 2018 13:39:10 +0000 Subject: [PATCH 054/110] Wazuh - Remove filebeat.yml --- salt/wazuh/files/filebeat.yml | 314 ---------------------------------- 1 file changed, 314 deletions(-) delete mode 100644 salt/wazuh/files/filebeat.yml diff --git a/salt/wazuh/files/filebeat.yml b/salt/wazuh/files/filebeat.yml deleted file mode 100644 index 7c6e0655b..000000000 --- a/salt/wazuh/files/filebeat.yml +++ /dev/null @@ -1,314 +0,0 @@ -{%- set MASTER = grains['master'] %} -{%- set HOSTNAME = salt['grains.get']('host', '') %} -{%- set BROVER = salt['pillar.get']('static:broversion', 'COMMUNITY') %} - -name: {{ HOSTNAME }} - -#========================== Modules configuration ============================ -filebeat.modules: -#=========================== Filebeat prospectors ============================= - -# List of prospectors to fetch data. -filebeat.prospectors: -#------------------------------ Log prospector -------------------------------- -{%- if BROVER != 'SURICATA' %} -{%- for LOGNAME in salt['pillar.get']('brologs:enabled', '') %} - - type: log - paths: - - /nsm/bro/logs/current/{{ LOGNAME }}.log - fields: - type: bro_{{ LOGNAME }} - fields_under_root: true - tags: ["bro"] - clean_removed: false - close_removed: false - -{%- endfor %} -{%- endif %} - - - type: log - paths: - - /suricata/eve.json - fields: - type: ids - engine: suricata - fields_under_root: true - clean_removed: false - close_removed: false - - - type: log - paths: - - /alerts/alerts.json - fields: - type: ossec - fields_under_root: true - clean_removed: false - close_removed: false - - - -#----------------------------- Logstash output --------------------------------- -output.logstash: - # Boolean flag to enable or disable the output module. - enabled: true - - # The Logstash hosts - hosts: ["{{ MASTER }}:5044"] - - # Number of workers per Logstash host. - worker: 1 - - # Set gzip compression level. - compression_level: 3 - - - # Enable SSL support. SSL is automatically enabled, if any SSL setting is set. - ssl.enabled: true - - # Configure SSL verification mode. If `none` is configured, all server hosts - # and certificates will be accepted. In this mode, SSL based connections are - # susceptible to man-in-the-middle attacks. Use only for testing. Default is - # `full`. - ssl.verification_mode: full - - # List of supported/valid TLS versions. By default all TLS versions 1.0 up to - # 1.2 are enabled. - #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] - - # Optional SSL configuration options. SSL is off by default. - # List of root certificates for HTTPS server verifications - ssl.certificate_authorities: ["/usr/share/filebeat/intraca.crt"] - - # Certificate for SSL client authentication - ssl.certificate: "/usr/share/filebeat/filebeat.crt" - - # Client Certificate Key - ssl.key: "/usr/share/filebeat/filebeat.key" - - -# Elasticsearch template settings -#setup.template.settings: - - # A dictionary of settings to place into the settings.index dictionary - # of the Elasticsearch template. For more details, please check - # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html - #index: - #number_of_shards: 1 - #codec: best_compression - #number_of_routing_shards: 30 - - # A dictionary of settings for the _source field. For more details, please check - # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html - #_source: - #enabled: false - -#============================== Kibana ===================================== - -# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. -# This requires a Kibana endpoint configuration. -#setup.kibana: - - # Kibana Host - # Scheme and port can be left out and will be set to the default (http and 5601) - # In case you specify and additional path, the scheme is required: http://localhost:5601/path - # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 - #host: "localhost:5601" - - # Optional protocol and basic auth credentials. - #protocol: "https" - #username: "elastic" - #password: "changeme" - - # Optional HTTP Path - #path: "" - - # Use SSL settings for HTTPS. Default is true. - #ssl.enabled: true - - # Configure SSL verification mode. If `none` is configured, all server hosts - # and certificates will be accepted. In this mode, SSL based connections are - # susceptible to man-in-the-middle attacks. Use only for testing. Default is - # `full`. - #ssl.verification_mode: full - - # List of supported/valid TLS versions. By default all TLS versions 1.0 up to - # 1.2 are enabled. - #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] - - # SSL configuration. By default is off. - # List of root certificates for HTTPS server verifications - #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - - # Certificate for SSL client authentication - #ssl.certificate: "/etc/pki/client/cert.pem" - - # Client Certificate Key - #ssl.key: "/etc/pki/client/cert.key" - - # Optional passphrase for decrypting the Certificate Key. - #ssl.key_passphrase: '' - - # Configure cipher suites to be used for SSL connections - #ssl.cipher_suites: [] - - # Configure curve types for ECDHE based cipher suites - #ssl.curve_types: [] - - - -#================================ Logging ====================================== -# There are four options for the log output: file, stderr, syslog, eventlog -# The file output is the default. - -# Sets log level. The default log level is info. -# Available log levels are: error, warning, info, debug -logging.level: debug - -# Enable debug output for selected components. To enable all selectors use ["*"] -# Other available selectors are "beat", "publish", "service" -# Multiple selectors can be chained. -#logging.selectors: [ ] - -# Send all logging output to syslog. The default is false. -#logging.to_syslog: false - -# Send all logging output to Windows Event Logs. The default is false. -#logging.to_eventlog: false - -# If enabled, filebeat periodically logs its internal metrics that have changed -# in the last period. For each metric that changed, the delta from the value at -# the beginning of the period is logged. Also, the total values for -# all non-zero internal metrics are logged on shutdown. The default is true. -#logging.metrics.enabled: true - -# The period after which to log the internal metrics. The default is 30s. -#logging.metrics.period: 30s - -# Logging to rotating files. Set logging.to_files to false to disable logging to -# files. -logging.to_files: true -logging.files: - # Configure the path where the logs are written. The default is the logs directory - # under the home path (the binary location). - path: /var/log/filebeat - - # The name of the files where the logs are written to. - name: filebeat - - # Configure log file size limit. If limit is reached, log file will be - # automatically rotated - rotateeverybytes: 10485760 # = 10MB - - # Number of rotated log files to keep. Oldest files will be deleted first. - keepfiles: 7 - - # The permissions mask to apply when rotating log files. The default value is 0600. - # Must be a valid Unix-style file permissions mask expressed in octal notation. - #permissions: 0600 - -# Set to true to log messages in json format. -#logging.json: false - - -#============================== Xpack Monitoring ===================================== -# filebeat can export internal metrics to a central Elasticsearch monitoring cluster. -# This requires xpack monitoring to be enabled in Elasticsearch. -# The reporting is disabled by default. - -# Set to true to enable the monitoring reporter. -#xpack.monitoring.enabled: false - -# Uncomment to send the metrics to Elasticsearch. Most settings from the -# Elasticsearch output are accepted here as well. Any setting that is not set is -# automatically inherited from the Elasticsearch output configuration, so if you -# have the Elasticsearch output configured, you can simply uncomment the -# following line, and leave the rest commented out. -#xpack.monitoring.elasticsearch: - - # Array of hosts to connect to. - # Scheme and port can be left out and will be set to the default (http and 9200) - # In case you specify and additional path, the scheme is required: http://localhost:9200/path - # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 - #hosts: ["localhost:9200"] - - # Set gzip compression level. - #compression_level: 0 - - # Optional protocol and basic auth credentials. - #protocol: "https" - #username: "beats_system" - #password: "changeme" - - # Dictionary of HTTP parameters to pass within the url with index operations. - #parameters: - #param1: value1 - #param2: value2 - - # Custom HTTP headers to add to each request - #headers: - # X-My-Header: Contents of the header - - # Proxy server url - #proxy_url: http://proxy:3128 - - # The number of times a particular Elasticsearch index operation is attempted. If - # the indexing operation doesn't succeed after this many retries, the events are - # dropped. The default is 3. - #max_retries: 3 - - # The maximum number of events to bulk in a single Elasticsearch bulk API index request. - # The default is 50. - #bulk_max_size: 50 - - # Configure http request timeout before failing an request to Elasticsearch. - #timeout: 90 - - # Use SSL settings for HTTPS. - #ssl.enabled: true - - # Configure SSL verification mode. If `none` is configured, all server hosts - # and certificates will be accepted. In this mode, SSL based connections are - # susceptible to man-in-the-middle attacks. Use only for testing. Default is - # `full`. - #ssl.verification_mode: full - - # List of supported/valid TLS versions. By default all TLS versions 1.0 up to - # 1.2 are enabled. - #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] - - # SSL configuration. By default is off. - # List of root certificates for HTTPS server verifications - #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - - # Certificate for SSL client authentication - #ssl.certificate: "/etc/pki/client/cert.pem" - - # Client Certificate Key - #ssl.key: "/etc/pki/client/cert.key" - - # Optional passphrase for decrypting the Certificate Key. - #ssl.key_passphrase: '' - - # Configure cipher suites to be used for SSL connections - #ssl.cipher_suites: [] - - # Configure curve types for ECDHE based cipher suites - #ssl.curve_types: [] - - # Configure what types of renegotiation are supported. Valid options are - # never, once, and freely. Default is never. - #ssl.renegotiation: never - -#================================ HTTP Endpoint ====================================== -# Each beat can expose internal metrics through a HTTP endpoint. For security -# reasons the endpoint is disabled by default. This feature is currently experimental. -# Stats can be access through http://localhost:5066/stats . For pretty JSON output -# append ?pretty to the URL. - -# Defines if the HTTP endpoint is enabled. -#http.enabled: false - -# The HTTP endpoint will bind to this hostname or IP address. It is recommended to use only localhost. -#http.host: localhost - -# Port on which the HTTP endpoint will bind. Default is 5066. From cdc8b577bd9038839f3df6535c38d29f680aa1a1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 7 Dec 2018 10:28:43 -0500 Subject: [PATCH 055/110] Redis Module - Update REDIS version to address vuln --- salt/redis/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/redis/init.sls b/salt/redis/init.sls index 81c47da93..cd982a137 100644 --- a/salt/redis/init.sls +++ b/salt/redis/init.sls @@ -49,7 +49,7 @@ toosmooth/so-redis:test2: so-redis: docker_container.running: - - image: soshybridhunter/so-redis:HH1.0.3 + - image: soshybridhunter/so-redis:HH1.0.5 - hostname: so-redis - user: socore - port_bindings: From e6469d505aa8931e8d6fe0e14ab44a2c0adff985 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 7 Dec 2018 18:13:42 +0000 Subject: [PATCH 056/110] Wazuh - initial init.sls --- salt/wazuh/init.sls | 128 ++++++++++++++++---------------------------- 1 file changed, 45 insertions(+), 83 deletions(-) diff --git a/salt/wazuh/init.sls b/salt/wazuh/init.sls index d034cab5c..1569d9933 100644 --- a/salt/wazuh/init.sls +++ b/salt/wazuh/init.sls @@ -1,91 +1,53 @@ -# Create a state directory +vm.max_map_count: + sysctl.present: + - value: 262144 -statedir: - file.directory: - - name: /opt/so/state +# Add ossec Group +ossecgroup: + group.present: + - name: ossec + - gid: 945 -salttmp: - file.directory: - - name: /opt/so/tmp +# Add ossecm user +ossecm: + user.present: + - uid: 943 + - gid: 945 + - home: /opt/so/wazuh + - createhome: False -# Install packages needed for the sensor +# Add ossecr user +ossecr: + user.present: + - uid: 944 + - gid: 945 + - home: /opt/so/wazuh + - createhome: False -sensorpkgs: - pkg.installed: - - skip_suggestions: True - - pkgs: - - docker-ce - - python-docker +# Add ossec user +ossec: + user.present: + - uid: 945 + - gid: 945 + - home: /opt/so/wazuh + - createhome: False -# Always keep these packages up to date +# Add wazuh agent +wazuhpkgs: + pkg.installed: + - skip_suggestions: False + - pkgs: + - wazuh-agent -alwaysupdated: - pkg.latest: - - pkgs: - - openssl - - openssh-server - - bash - - skip_suggestions: True - -# Set time to UTC - -Etc/UTC: - timezone.system - -# Set up docker network -dockernet: - docker_network.present: - - name: so-elastic-net - - driver: bridge - -# Snag the so-core docker -toosmooth/so-core:test2: - docker_image.present - -# Drop the correct nginx config based on role - -nginxconfdir: - file.directory: - - name: /opt/so/conf/nginx - - user: 939 - - group: 939 - - makedirs: True - -nginxconf: - file.managed: - - name: /opt/so/conf/nginx/nginx.conf - - user: 939 - - group: 939 - - template: jinja - - source: salt://common/nginx/nginx.conf.{{ grains.role }} - -nginxlogdir: - file.directory: - - name: /opt/so/log/nginx/ - - user: 939 - - group: 939 - -nginxtmp: - file.directory: - - name: /opt/so/tmp/nginx/tmp - - user: 939 - - group: 939 - - makedirs: True - -# Start the core docker -so-core: +so-wazuh: docker_container.running: - - image: toosmooth/so-core:test2 - - hostname: so-core - - user: socore - - binds: - - /opt/so:/opt/so:rw - - /opt/so/conf/nginx/nginx.conf:/etc/nginx/nginx.conf:ro - - /opt/so/log/nginx/:/var/log/nginx:rw - - /opt/so/tmp/nginx/:/var/lib/nginx:rw - - /opt/so/tmp/nginx/:/run:rw - - network_mode: so-elastic-net - - cap_add: NET_BIND_SERVICE + - image: soshybridhunter/so-wazuh:HH1.0.5 + - hostname: {{ hostname}}-docker + - name: so-wazuh + - user: ossec - port_bindings: - - 80:80 - - 443:443 + - 0.0.0.0:1514:1514 + - 0.0.0.0:55000:55000 + - binds: + - /opt/so/wazuh/:/var/ossec/data:rw + From 0a332047260d405d3b63b30a1a804c4873937914 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 7 Dec 2018 18:16:03 +0000 Subject: [PATCH 057/110] Setup - Only add Wazuh repo --- so-setup-network.sh | 52 ++++++++++++++++++--------------------------- 1 file changed, 21 insertions(+), 31 deletions(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index 240172838..e1e98dce7 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -77,16 +77,17 @@ add_socore_user_notmaster() { add_wazuh_users() { - if [ $OS == 'centos' ]; then - local ADDUSER=adduser - else - local ADDUSER=useradd - fi + # REMARKING FOR NOW -- ADDING VIA init.sls + #if [ $OS == 'centos' ]; then + # local ADDUSER=adduser + #else + # local ADDUSER=useradd + #fi - groupadd --gid 945 ossec - $ADDUSER --uid 943 --gid 945 --home-dir /opt/so/wazuh --no-create-home ossecm - $ADDUSER --uid 944 --gid 945 --home-dir /opt/so/wazuh --no-create-home ossecr - $ADDUSER --uid 945 --gid 945 --home-dir /opt/so/wazuh --no-create-home ossec + #groupadd --gid 945 ossec + #$ADDUSER --uid 943 --gid 945 --home-dir /opt/so/wazuh --no-create-home ossecm + #$ADDUSER --uid 944 --gid 945 --home-dir /opt/so/wazuh --no-create-home ossecr + #$ADDUSER --uid 945 --gid 945 --home-dir /opt/so/wazuh --no-create-home ossec } @@ -891,7 +892,7 @@ update_sudoers() { } -wazuh_agent_install() { +wazuh_repo_install() { if [ $OS == 'centos' ]; then # Add repo @@ -904,22 +905,11 @@ name=Wazuh repository baseurl=https://packages.wazuh.com/3.x/yum/ protect=1 EOF - # Install agent - yum install -y wazuh-agent - # Prevent automatic upates - sed -i "s/^enabled=1/enabled=0/" /etc/yum.repos.d/wazuh.repo else # Get key curl -s https://packages.wazuh.com/key/GPG-KEY-WAZUH | apt-key add - # Add repo echo "deb https://packages.wazuh.com/3.x/apt/ stable main" | tee /etc/apt/sources.list.d/wazuh.list - apt-get update -y - # Install - apt-get install -y wazuh-agent - # Prevent automatic updates - sed -i "s/^deb/#deb/" /etc/apt/sources.list.d/wazuh.list - # Set package state to "hold" - echo "wazuh-agent hold" | sudo dpkg --set-selections fi } @@ -1470,11 +1460,11 @@ if (whiptail_you_sure); then echo "" add_socore_user_master - echo "** Adding Wazuh users **" - add_wazuh_users + #echo "** Adding Wazuh users **" + #add_wazuh_users - echo "** Installing Wazuh agent **" - wazuh_agent_install + echo "** Installing Wazuh repo **" + wazuh_repo_install # Install salt and dependencies echo " ** Installing Salt and Dependencies **" @@ -1562,8 +1552,8 @@ if (whiptail_you_sure); then mkdir -p /nsm get_filesystem_root get_filesystem_nsm - add_wazuh_users - wazuh_agent_install + #add_wazuh_users + wazuh_repo_install copy_ssh_key set_initial_firewall_policy create_bond @@ -1628,8 +1618,8 @@ if (whiptail_you_sure); then echo "**** Please set a password for socore. You will use this password when setting up other Nodes/Sensors" echo "" add_socore_user_master - add_wazuh_users - wazuh_agent_install + #add_wazuh_users + wazuh_repo_install create_bond saltify docker_install @@ -1691,8 +1681,8 @@ if (whiptail_you_sure); then mkdir -p /nsm get_filesystem_root get_filesystem_nsm - add_wazuh_users - wazuh_agent_install + #add_wazuh_users + wazuh_repo_install copy_ssh_key set_initial_firewall_policy saltify From cb68f502ee3f0925b72e699eda315ae9a71fbaae Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Mon, 10 Dec 2018 19:49:14 +0000 Subject: [PATCH 058/110] Wazuh - Changes to init.sls --- salt/wazuh/init.sls | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/salt/wazuh/init.sls b/salt/wazuh/init.sls index 1569d9933..a7f06ab33 100644 --- a/salt/wazuh/init.sls +++ b/salt/wazuh/init.sls @@ -1,6 +1,8 @@ -vm.max_map_count: - sysctl.present: - - value: 262144 +{%- set HOSTNAME = salt['grains.get']('host', '') %} + +#vm.max_map_count: +# sysctl.present: +# - value: 262144 # Add ossec Group ossecgroup: @@ -42,12 +44,12 @@ wazuhpkgs: so-wazuh: docker_container.running: - image: soshybridhunter/so-wazuh:HH1.0.5 - - hostname: {{ hostname}}-docker + - hostname: {{HOSTNAME}}-wazuh-manager - name: so-wazuh - - user: ossec + - detach: True - port_bindings: - 0.0.0.0:1514:1514 - 0.0.0.0:55000:55000 - binds: - - /opt/so/wazuh/:/var/ossec/data:rw + - /opt/so/wazuh/:/var/ossec/data/:rw From e70db05a0f044684e87ea3d834e8d81fea3edf49 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Mon, 10 Dec 2018 19:50:55 +0000 Subject: [PATCH 059/110] Filebeat - Modify config for Wazuh alerts --- salt/filebeat/etc/filebeat.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 7c6e0655b..f0bbe3e11 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -1,6 +1,7 @@ {%- set MASTER = grains['master'] %} {%- set HOSTNAME = salt['grains.get']('host', '') %} {%- set BROVER = salt['pillar.get']('static:broversion', 'COMMUNITY') %} +{%- set WAZUHENABLED = salt['pillar.get']('static:wazuh_enabled', '1') %} name: {{ HOSTNAME }} @@ -36,16 +37,16 @@ filebeat.prospectors: clean_removed: false close_removed: false +{%- if WAZUHENABLED != '1' %} - type: log paths: - - /alerts/alerts.json + - /wazuh/alerts/alerts.json fields: type: ossec fields_under_root: true clean_removed: false close_removed: false - - +{%- endif %} #----------------------------- Logstash output --------------------------------- output.logstash: From 25449844339bd68c74bfcbc1ca82cd1f2a3da672 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Mon, 10 Dec 2018 19:51:57 +0000 Subject: [PATCH 060/110] Wazuh - add to top.sls for Eval Mode --- salt/top.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/top.sls b/salt/top.sls index 05e027a84..413a120f6 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -33,6 +33,7 @@ base: - utility - schedule - fleet + - wazuh 'G@role:so-master': From 10d6c0f5a9d4901383dfadef36db67ee237fcac5 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Mon, 10 Dec 2018 19:55:53 +0000 Subject: [PATCH 061/110] Setup - Remark Wazuh user section completely --- so-setup-network.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index e1e98dce7..6fe2222ae 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -75,7 +75,7 @@ add_socore_user_notmaster() { } -add_wazuh_users() { +#add_wazuh_users() { # REMARKING FOR NOW -- ADDING VIA init.sls #if [ $OS == 'centos' ]; then @@ -89,7 +89,7 @@ add_wazuh_users() { #$ADDUSER --uid 944 --gid 945 --home-dir /opt/so/wazuh --no-create-home ossecr #$ADDUSER --uid 945 --gid 945 --home-dir /opt/so/wazuh --no-create-home ossec -} +#} # Enable Bro Logs From 0f5fbadaf59ca3b2e459a7c5901fbf7fc4dccde4 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Mon, 10 Dec 2018 20:17:41 +0000 Subject: [PATCH 062/110] Filebeat - Switch negation to equals --- salt/filebeat/etc/filebeat.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index f0bbe3e11..342b925a0 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -37,7 +37,8 @@ filebeat.prospectors: clean_removed: false close_removed: false -{%- if WAZUHENABLED != '1' %} +{%- if WAZUHENABLED == '1' %} + - type: log paths: - /wazuh/alerts/alerts.json From 8c1a7b3e0ca4a41f86abfb819021490b20bfc56c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 11 Dec 2018 11:19:54 -0500 Subject: [PATCH 063/110] Setup - Change so passwords survive re-install --- pillar/top.sls | 2 ++ salt/fleet/init.sls | 4 ++-- salt/mysql/init.sls | 4 ++-- so-setup-network.sh | 28 +++++++++++++++++++++++----- 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/pillar/top.sls b/pillar/top.sls index b9117b19c..bc68aa644 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -10,6 +10,7 @@ base: - static - firewall.* - data.* + - auth 'G@role:so-eval': - masters.{{ grains.host }} @@ -17,6 +18,7 @@ base: - firewall.* - data.* - brologs + - auth 'G@role:so-node': - nodes.{{ grains.host }} diff --git a/salt/fleet/init.sls b/salt/fleet/init.sls index 7c1f0099d..a90377b1f 100644 --- a/salt/fleet/init.sls +++ b/salt/fleet/init.sls @@ -1,5 +1,5 @@ -{%- set MYSQLPASS = salt['pillar.get']('master:mysqlpass', 'iwonttellyou') %} -{%- set FLEETPASS = salt['pillar.get']('master:fleetpass', 'bazinga') -%} +{%- set MYSQLPASS = salt['pillar.get']('auth:mysql', 'iwonttellyou') %} +{%- set FLEETPASS = salt['pillar.get']('auth:fleet', 'bazinga') -%} {%- set MASTERIP = salt['pillar.get']('static:masterip', '') -%} # Fleet Setup diff --git a/salt/mysql/init.sls b/salt/mysql/init.sls index 785e2702a..af80030ee 100644 --- a/salt/mysql/init.sls +++ b/salt/mysql/init.sls @@ -1,5 +1,5 @@ -{%- set MYSQLPASS = salt['pillar.get']('master:mysqlpass', 'iwonttellyou') %} -{%- set FLEETPASS = salt['pillar.get']('master:fleetpass', 'bazinga') %} +{%- set MYSQLPASS = salt['pillar.get']('auth:mysql', 'iwonttellyou') %} +{%- set FLEETPASS = salt['pillar.get']('auth:fleet', 'bazinga') %} {%- set MASTERIP = salt['pillar.get']('static:masterip', '') %} # MySQL Setup mysqlpkgs: diff --git a/so-setup-network.sh b/so-setup-network.sh index 6fe2222ae..f05addfca 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -76,7 +76,7 @@ add_socore_user_notmaster() { } #add_wazuh_users() { - + # REMARKING FOR NOW -- ADDING VIA init.sls #if [ $OS == 'centos' ]; then # local ADDUSER=adduser @@ -91,6 +91,18 @@ add_socore_user_notmaster() { #} +# Create an auth pillar so that passwords survive re-install +auth_pillar(){ + + if [ ! -f /opt/so/saltstack/pillar/auth.sls ]; then + echo "Creating Auth Pillar" + mkdir -p /opt/so/saltstack/pillar + echo "auth:" >> /opt/so/saltstack/pillar/auth.sls + echo " mysql: $MYSQLPASS" >> /opt/so/saltstack/pillar/auth.sls + echo " fleet: $FLEETPASS" >> /opt/so/saltstack/pillar/auth.sls + fi + +} # Enable Bro Logs bro_logs_enabled() { @@ -192,7 +204,12 @@ configure_minion() { echo "mysql.host: '$MAINIP'" >> /etc/salt/minion echo "mysql.port: 3306" >> /etc/salt/minion echo "mysql.user: 'root'" >> /etc/salt/minion - echo "mysql.pass: '$MYSQLPASS'" >> /etc/salt/minion + if [ ! -f /opt/so/saltstack/pillar/auth.sls ]; then + echo "mysql.pass: '$MYSQLPASS'" >> /etc/salt/minion + else + OLDPASS=$(cat /opt/so/saltstack/pillar/auth.sls | grep mysql | awk {'print $2'}) + echo "mysql.pass: '$OLDPASS'" >> /etc/salt/minion + fi else echo "master: $MSRV" > /etc/salt/minion echo "id: $HOSTNAME" >> /etc/salt/minion @@ -531,8 +548,8 @@ master_pillar() { echo " es_port: $NODE_ES_PORT" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls echo " log_size_limit: $LOG_SIZE_LIMIT" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls echo " cur_close_days: $CURCLOSEDAYS" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls - echo " mysqlpass: $MYSQLPASS" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls - echo " fleetpass: $FLEETPASS" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls + #echo " mysqlpass: $MYSQLPASS" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls + #echo " fleetpass: $FLEETPASS" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls } @@ -893,7 +910,7 @@ update_sudoers() { } wazuh_repo_install() { - + if [ $OS == 'centos' ]; then # Add repo cat > /etc/yum.repos.d/wazuh.repo <<\EOF @@ -1444,6 +1461,7 @@ if (whiptail_you_sure); then # Last Chance to back out whiptail_make_changes generate_passwords + auth_pillar clear_master mkdir -p /nsm get_filesystem_root From a54a5ede8c277fd760c402e386f37f29021a106b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 11 Dec 2018 11:32:37 -0500 Subject: [PATCH 064/110] MySQL Module - fix password designation --- salt/mysql/etc/mypass | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/mysql/etc/mypass b/salt/mysql/etc/mypass index dad35b5e3..85ae1772b 100644 --- a/salt/mysql/etc/mypass +++ b/salt/mysql/etc/mypass @@ -1,2 +1,2 @@ -{%- set MYSQLPASS = salt['pillar.get']('master:mysqlpass', 'iwonttellyou') -%} +{%- set MYSQLPASS = salt['pillar.get']('auth:mysql', 'iwonttellyou') -%} {{ MYSQLPASS }} From bea42860547dc41623c9f31893e20448fe5e7352 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 11 Dec 2018 11:57:27 -0500 Subject: [PATCH 065/110] Setup - Fix static file creation --- so-setup-network.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index f05addfca..0f563a4b3 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -558,7 +558,7 @@ master_static() { # Create a static file for global values touch /opt/so/saltstack/pillar/static.sls - echo "static:" >> /opt/so/saltstack/pillar/static.sls + echo "static:" > /opt/so/saltstack/pillar/static.sls echo " hnmaster: $HNMASTER" >> /opt/so/saltstack/pillar/static.sls echo " ntpserver: $NTPSERVER" >> /opt/so/saltstack/pillar/static.sls echo " proxy: $PROXY" >> /opt/so/saltstack/pillar/static.sls From dd15a6e31a94a676620af951ddaaaa4ebd6397b9 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 11 Dec 2018 12:27:57 -0500 Subject: [PATCH 066/110] SSL Module - Fixed it so certs do not keep renewing --- salt/ssl/init.sls | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 12ab3a9c7..f20988694 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -23,7 +23,8 @@ m2cryptopkgs: - signing_policy: influxdb - public_key: /etc/pki/influxdb.key - CN: {{ master }} - - days_remaining: 3000 + - days_remaining: 0 + - days_valid: 3650 - backup: True - managed_private_key: name: /etc/pki/influxdb.key @@ -39,7 +40,8 @@ m2cryptopkgs: - signing_policy: filebeat - public_key: /etc/pki/filebeat.key - CN: {{ master }} - - days_remaining: 3000 + - days_remaining: 0 + - days_valid: 3650 - backup: True - managed_private_key: name: /etc/pki/filebeat.key @@ -71,7 +73,8 @@ fbcrtlink: - signing_policy: registry - public_key: /etc/pki/registry.key - CN: {{ master }} - - days_remaining: 3000 + - days_remaining: 0 + - days_valid: 3650 - backup: True - managed_private_key: name: /etc/pki/registry.key @@ -85,7 +88,8 @@ fbcrtlink: - signing_policy: masterssl - public_key: /etc/pki/masterssl.key - CN: {{ master }} - - days_remaining: 3000 + - days_remaining: 0 + - days_valid: 3650 - backup: True - managed_private_key: name: /etc/pki/masterssl.key @@ -99,7 +103,8 @@ fbcrtlink: - signing_policy: fleet - public_key: /etc/pki/fleet.key - CN: {{ master }} - - days_remaining: 3000 + - days_remaining: 0 + - days_valid: 3650 - backup: True - managed_private_key: name: /etc/pki/fleet.key @@ -121,7 +126,8 @@ fbcertdir: - signing_policy: filebeat - public_key: /opt/so/conf/filebeat/etc/pki/filebeat.key - CN: {{ master }} - - days_remaining: 3000 + - days_remaining: 0 + - days_valid: 3650 - backup: True - managed_private_key: name: /opt/so/conf/filebeat/etc/pki/filebeat.key From 6cdf1ef857a961210f22cd51944e38469bd3c197 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 11 Dec 2018 19:44:32 +0000 Subject: [PATCH 067/110] Firewall - Add rules for Wazuh Manager --- salt/firewall/init.sls | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/salt/firewall/init.sls b/salt/firewall/init.sls index 71575e3d6..ef2acd81f 100644 --- a/salt/firewall/init.sls +++ b/salt/firewall/init.sls @@ -1,5 +1,19 @@ # Firewall Magic for the grid +{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' %} + +{%- set ip = salt['pillar.get']('static:masterip', '') %} + +{% elif grains['role'] == 'so-node'%} + +{%- set ip = salt['pillar.get']('node:mainip', '') %} + +{% elif grains['role'] == 'so-sensor'%} + +{%- set ip = salt['pillar.get']('node:mainip', '') %} + +{% endif %} + # Keep localhost in the game iptables_allow_localhost: iptables.append: @@ -86,6 +100,29 @@ enable_docker_user_established: - match: conntrack - ctstate: 'RELATED,ESTABLISHED' +# Add rule(s) for Wazuh manager +enable_wazuh_manager_1514_tcp_{{ip}}: + iptables.insert: + - table: filter + - chain: DOCKER-USER + - jump: ACCEPT + - proto: tcp + - source: {{ ip }} + - dport: 1514 + - position: 1 + - save: True + +enable_wazuh_manager_1514_udp_{{ip}}: + iptables.insert: + - table: filter + - chain: DOCKER-USER + - jump: ACCEPT + - proto: udp + - source: {{ ip }} + - dport: 1514 + - position: 1 + - save: True + # Rules if you are a Master {% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' %} #This should be more granular From 223237f8c25a31f0330a352e68fe9a2fe4aaa750 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 11 Dec 2018 19:45:56 +0000 Subject: [PATCH 068/110] Wazuh - Expose both UDP and TCP ports --- salt/wazuh/init.sls | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/wazuh/init.sls b/salt/wazuh/init.sls index a7f06ab33..622ef20e8 100644 --- a/salt/wazuh/init.sls +++ b/salt/wazuh/init.sls @@ -48,7 +48,8 @@ so-wazuh: - name: so-wazuh - detach: True - port_bindings: - - 0.0.0.0:1514:1514 + - 0.0.0.0:1515:1514/udp + - 0.0.0.0:1514:1514/tcp - 0.0.0.0:55000:55000 - binds: - /opt/so/wazuh/:/var/ossec/data/:rw From 9a021164ace4824cfef6976b2caf71d567b2b241 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 12 Dec 2018 01:42:05 +0000 Subject: [PATCH 069/110] Wazuh - Fix port, add agent conf, and agent registration script --- salt/wazuh/files/agent/ossec.conf | 195 ++++++++++++++++++++ salt/wazuh/files/agent/wazuh-register-agent | 131 +++++++++++++ salt/wazuh/init.sls | 11 +- 3 files changed, 336 insertions(+), 1 deletion(-) create mode 100644 salt/wazuh/files/agent/ossec.conf create mode 100755 salt/wazuh/files/agent/wazuh-register-agent diff --git a/salt/wazuh/files/agent/ossec.conf b/salt/wazuh/files/agent/ossec.conf new file mode 100644 index 000000000..c89b9ce06 --- /dev/null +++ b/salt/wazuh/files/agent/ossec.conf @@ -0,0 +1,195 @@ +{%- if grains['role'] == 'so-master' or grains['role'] == 'so-eval' %} +{%- set ip = salt['pillar.get']('static:masterip', '') %} +{%- endif %} + + + + + +
{{ip}}
+ 1514 + udp + + ubuntu, ubuntu16, ubuntu16.04 + 10 + 60 + yes + aes + + + + + no + 5000 + 500 + + + + + no + yes + yes + yes + yes + yes + yes + yes + yes + + + 43200 + + /var/ossec/etc/shared/rootkit_files.txt + /var/ossec/etc/shared/rootkit_trojans.txt + + /var/ossec/etc/shared/system_audit_rcl.txt + /var/ossec/etc/shared/system_audit_ssh.txt + + yes + + + + yes + 1800 + 1d + yes + + + + yes + 1800 + 1d + yes + + wodles/java + wodles/ciscat + + + + + yes + yes + /var/log/osquery/osqueryd.results.log + /etc/osquery/osquery.conf + yes + + + + + no + 1h + yes + yes + yes + yes + yes + yes + yes + + + + + no + + + 43200 + + yes + + + /etc,/usr/bin,/usr/sbin + /bin,/sbin,/boot + + + /etc/mtab + /etc/hosts.deny + /etc/mail/statistics + /etc/random-seed + /etc/random.seed + /etc/adjtime + /etc/httpd/logs + /etc/utmpx + /etc/wtmpx + /etc/cups/certs + /etc/dumpdates + /etc/svc/volatile + /sys/kernel/security + /sys/kernel/debug + + + /etc/ssl/private.key + + yes + + + yes + + + yes + + + + + command + df -P + 360 + + + + full_command + netstat -tulpn | sed 's/\([[:alnum:]]\+\)\ \+[[:digit:]]\+\ \+[[:digit:]]\+\ \+\(.*\):\([[:digit:]]*\)\ \+\([0-9\.\:\*]\+\).\+\ \([[:digit:]]*\/[[:alnum:]\-]*\).*/\1 \2 == \3 == \4 \5/' | sort -k 4 -g | sed 's/ == \(.*\) ==/:\1/' | sed 1,2d + netstat listening ports + 360 + + + + full_command + last -n 20 + 360 + + + + + no + /var/ossec/etc/wpk_root.pem + yes + + + + + plain + + + + + + + syslog + /var/ossec/logs/active-responses.log + + + + syslog + /var/log/auth.log + + + + syslog + /var/log/syslog + + + + syslog + /var/log/dpkg.log + + + + syslog + /var/log/kern.log + + + diff --git a/salt/wazuh/files/agent/wazuh-register-agent b/salt/wazuh/files/agent/wazuh-register-agent new file mode 100755 index 000000000..e9f9dbeb5 --- /dev/null +++ b/salt/wazuh/files/agent/wazuh-register-agent @@ -0,0 +1,131 @@ +#!/bin/bash + +### +# Shell script for registering agents automatically with the API +# Copyright (C) 2017 Wazuh, Inc. All rights reserved. +# Wazuh.com +# +# This program is a free software; you can redistribute it +# and/or modify it under the terms of the GNU General Public +# License (version 2) as published by the FSF - Free Software +# Foundation. +### +# +# 12/11/2018 +# This script has been modified by Security Onion Solutions +# - Added Agent IP variable and option +### + +# Connection variables +API_IP="localhost" +API_PORT="55000" +PROTOCOL="https" +USER="foo" +PASSWORD="bar" +AGENT_NAME=$(hostname) +AGENT_IP="" + +display_help() { +cat < agent is not registered +# if ! [ "$AGENT_ID" -eq "$AGENT_ID" ] 2> /dev/null ; then +# echo "Starting registration process ..." +# : +# elif [[ "$FORCE" = true && "$SILENT" = "true" ]] ; then +# remove_agent > /dev/null 2>&1 +# else +# if [[ "$FORCE" = true ]] ; then +# remove_agent +# fi +# fi + +# Default action -> try to register the agent +register_agent +#remove_agent diff --git a/salt/wazuh/init.sls b/salt/wazuh/init.sls index 622ef20e8..2dace4cac 100644 --- a/salt/wazuh/init.sls +++ b/salt/wazuh/init.sls @@ -41,6 +41,15 @@ wazuhpkgs: - pkgs: - wazuh-agent +# Add Wazuh agent conf +eslog4jfile: + file.managed: + - name: /var/ossec/etc/ossec.conf + - source: salt://wazuh/files/agent/ossec.conf + - user: 0 + - group: 945 + - template: jinja + so-wazuh: docker_container.running: - image: soshybridhunter/so-wazuh:HH1.0.5 @@ -48,7 +57,7 @@ so-wazuh: - name: so-wazuh - detach: True - port_bindings: - - 0.0.0.0:1515:1514/udp + - 0.0.0.0:1514:1514/udp - 0.0.0.0:1514:1514/tcp - 0.0.0.0:55000:55000 - binds: From 634c435ad60d4aab737a7fc652834cc25c3bdd79 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 12 Dec 2018 01:51:30 +0000 Subject: [PATCH 070/110] Setup - Configure Wazuh agent --- so-setup-network.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/so-setup-network.sh b/so-setup-network.sh index 0f563a4b3..77d77fbd5 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -220,6 +220,14 @@ configure_minion() { } +configure_wazuh_agent(){ + + # Configure Wazuh agent to talk to manager + echo "Configuring Wazuh agent to talk to manager..." + /usr/sbin/wazuh-register-agent -i $MAINIP + +} + copy_master_config() { # Copy the master config template to the proper directory @@ -1660,6 +1668,7 @@ if (whiptail_you_sure); then salt_checkin_message salt_checkin checkin_at_boot + configure_wazuh_agent whiptail_setup_complete fi From 113f03087333fc5d15565f5fb4c8f7d66c0cf9e8 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 12 Dec 2018 02:26:38 +0000 Subject: [PATCH 071/110] Wazuh - Add agent register script to init.sls --- salt/wazuh/init.sls | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/salt/wazuh/init.sls b/salt/wazuh/init.sls index 2dace4cac..335f29bc9 100644 --- a/salt/wazuh/init.sls +++ b/salt/wazuh/init.sls @@ -42,7 +42,7 @@ wazuhpkgs: - wazuh-agent # Add Wazuh agent conf -eslog4jfile: +wazuhagentconf: file.managed: - name: /var/ossec/etc/ossec.conf - source: salt://wazuh/files/agent/ossec.conf @@ -50,6 +50,14 @@ eslog4jfile: - group: 945 - template: jinja +# Add Wazuh agent conf +wazuhagentregister: + file.managed: + - name: /usr/sbin/wazuh-agent-register + - source: salt://wazuh/files/agent/wazuh-register-agent + - user: 0 + - group: 0 + so-wazuh: docker_container.running: - image: soshybridhunter/so-wazuh:HH1.0.5 From 86a72984c76fe15331bec58caf138a5ab0e9f301 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 12 Dec 2018 02:58:09 +0000 Subject: [PATCH 072/110] Setup - Add auth pillar to eval mode --- so-setup-network.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/so-setup-network.sh b/so-setup-network.sh index 77d77fbd5..065847fac 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -1633,6 +1633,7 @@ if (whiptail_you_sure); then CURCLOSEDAYS=30 whiptail_make_changes generate_passwords + auth_pillar clear_master mkdir -p /nsm get_filesystem_root From 1a4a7382e254257308344a8d3b07332a84ff402e Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 12 Dec 2018 03:18:55 +0000 Subject: [PATCH 073/110] Wazuh - Fix Wazuh agent registration script name --- salt/wazuh/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/wazuh/init.sls b/salt/wazuh/init.sls index 335f29bc9..ff6de8b84 100644 --- a/salt/wazuh/init.sls +++ b/salt/wazuh/init.sls @@ -53,7 +53,7 @@ wazuhagentconf: # Add Wazuh agent conf wazuhagentregister: file.managed: - - name: /usr/sbin/wazuh-agent-register + - name: /usr/sbin/wazuh-register-agent - source: salt://wazuh/files/agent/wazuh-register-agent - user: 0 - group: 0 From 823a589fae2631388d24bdee461d6bc8c10dba40 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 12 Dec 2018 04:01:13 +0000 Subject: [PATCH 074/110] Wazuh - Set mode for agent registration script --- salt/wazuh/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/wazuh/init.sls b/salt/wazuh/init.sls index ff6de8b84..1d0b9a99e 100644 --- a/salt/wazuh/init.sls +++ b/salt/wazuh/init.sls @@ -57,6 +57,7 @@ wazuhagentregister: - source: salt://wazuh/files/agent/wazuh-register-agent - user: 0 - group: 0 + - mode: 755 so-wazuh: docker_container.running: From 8404897fe3cf7fe9dbd66b15552e8a995bbf3b05 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 12 Dec 2018 06:05:13 +0000 Subject: [PATCH 075/110] Wazuh - Move agent config to init.sls --- salt/wazuh/files/agent/wazuh-register-agent | 5 +++-- salt/wazuh/init.sls | 13 ++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/salt/wazuh/files/agent/wazuh-register-agent b/salt/wazuh/files/agent/wazuh-register-agent index e9f9dbeb5..1854f55ff 100755 --- a/salt/wazuh/files/agent/wazuh-register-agent +++ b/salt/wazuh/files/agent/wazuh-register-agent @@ -1,3 +1,4 @@ +{%- set ip = salt['pillar.get']('static:masterip', '') %} #!/bin/bash ### @@ -23,7 +24,7 @@ PROTOCOL="https" USER="foo" PASSWORD="bar" AGENT_NAME=$(hostname) -AGENT_IP="" +AGENT_IP="{{ip}}" display_help() { cat < Date: Wed, 12 Dec 2018 13:10:27 +0000 Subject: [PATCH 076/110] Setup - Remark Wazuh agent config --- so-setup-network.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index 065847fac..72496657d 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -1669,7 +1669,7 @@ if (whiptail_you_sure); then salt_checkin_message salt_checkin checkin_at_boot - configure_wazuh_agent + #configure_wazuh_agent whiptail_setup_complete fi From 5822842d2e874f5b13b21924e966f7b1523ecfd4 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 12 Dec 2018 13:36:13 +0000 Subject: [PATCH 077/110] Wazuh - Add sleep to wait for API --- salt/wazuh/files/agent/wazuh-register-agent | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/wazuh/files/agent/wazuh-register-agent b/salt/wazuh/files/agent/wazuh-register-agent index 1854f55ff..b6199cf9a 100755 --- a/salt/wazuh/files/agent/wazuh-register-agent +++ b/salt/wazuh/files/agent/wazuh-register-agent @@ -128,5 +128,6 @@ shift $(($OPTIND - 1)) # fi # Default action -> try to register the agent +sleep 10s register_agent #remove_agent From e20ab3b4073051febe20d76a843e36b87112434e Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 12 Dec 2018 14:48:17 +0000 Subject: [PATCH 078/110] Filebeat - Config for Wazuh alerts --- salt/filebeat/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 251274606..7563ad72a 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -61,6 +61,7 @@ so-filebeat: - /opt/so/conf/filebeat/etc/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro - /nsm/bro:/nsm/bro:ro - /opt/so/log/suricata:/suricata:ro + - /opt/so/wazuh/alerts/alerts.json:/wazuh/alerts/alerts.json:ro - /opt/so/conf/filebeat/etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro - /opt/so/conf/filebeat/etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro - /etc/ssl/certs/intca.crt:/usr/share/filebeat/intraca.crt:ro From 9d86744e076dd38ff01bd566afa3bf1ec02df29a Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 12 Dec 2018 15:19:51 +0000 Subject: [PATCH 079/110] Filebeat - Fix Wazuh alerts path --- salt/filebeat/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 7563ad72a..8b0ec3f4c 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -61,7 +61,7 @@ so-filebeat: - /opt/so/conf/filebeat/etc/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro - /nsm/bro:/nsm/bro:ro - /opt/so/log/suricata:/suricata:ro - - /opt/so/wazuh/alerts/alerts.json:/wazuh/alerts/alerts.json:ro + - /opt/so/wazuh/logs/alerts/:/wazuh/alerts:ro - /opt/so/conf/filebeat/etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro - /opt/so/conf/filebeat/etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro - /etc/ssl/certs/intca.crt:/usr/share/filebeat/intraca.crt:ro From 8496834f8bc539e3075944fcc09938d1f7c9b768 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 12 Dec 2018 15:48:59 +0000 Subject: [PATCH 080/110] Wazuh - Re-order top.sls so Filebeat does not overrite Wazuh logs --- salt/top.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/top.sls b/salt/top.sls index 413a120f6..03c220047 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -29,11 +29,11 @@ base: - bro - curator - elastalert + - fleet + - wazuh - filebeat - utility - schedule - - fleet - - wazuh 'G@role:so-master': From 54c35cdc0dc1e9fb2ac0d35f65cef5009aed7d34 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 12 Dec 2018 20:51:41 +0000 Subject: [PATCH 081/110] Filebeat - Add Wazuh archive logs --- salt/filebeat/etc/filebeat.yml | 10 ++++++++++ salt/filebeat/init.sls | 1 + 2 files changed, 11 insertions(+) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 342b925a0..4384d124e 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -47,6 +47,16 @@ filebeat.prospectors: fields_under_root: true clean_removed: false close_removed: false + + - type: log + paths: + - /wazuh/archives/archives.json + fields: + type: ossec_archive + fields_under_root: true + clean_removed: false + close_removed: false + {%- endif %} #----------------------------- Logstash output --------------------------------- diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 8b0ec3f4c..da8f0637c 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -62,6 +62,7 @@ so-filebeat: - /nsm/bro:/nsm/bro:ro - /opt/so/log/suricata:/suricata:ro - /opt/so/wazuh/logs/alerts/:/wazuh/alerts:ro + - /opt/so/wazuh/logs/archives/:/wazuh/archives:ro - /opt/so/conf/filebeat/etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro - /opt/so/conf/filebeat/etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro - /etc/ssl/certs/intca.crt:/usr/share/filebeat/intraca.crt:ro From 41e9c4c7e0caf0fbe3fe3860d740c8510179cbfe Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 12 Dec 2018 20:52:18 +0000 Subject: [PATCH 082/110] Logstash - Alter input for Wazuh logs --- salt/logstash/files/dynamic/0006_input_beats.conf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/salt/logstash/files/dynamic/0006_input_beats.conf b/salt/logstash/files/dynamic/0006_input_beats.conf index b263e611b..bd41a3024 100644 --- a/salt/logstash/files/dynamic/0006_input_beats.conf +++ b/salt/logstash/files/dynamic/0006_input_beats.conf @@ -9,7 +9,7 @@ input { } } filter { - if [type] == "ids" { + if [type] == "ids" or [type] =~ "bro" { mutate { rename => { "host" => "beat_host" } remove_tag => ["beat"] @@ -17,11 +17,10 @@ filter { add_field => { "syslog-host_from" => "%{[beat][name]}" } } } - if "bro" in [tags] { + if [type] =~ "ossec" { mutate { rename => { "host" => "beat_host" } remove_tag => ["beat"] - add_field => { "sensor_name" => "%{[beat][name]}" } add_field => { "syslog-host_from" => "%{[beat][name]}" } } } From 5c737e9fda6340572ce3912ab034dba1143c14a0 Mon Sep 17 00:00:00 2001 From: dlee35 Date: Wed, 12 Dec 2018 16:19:35 -0500 Subject: [PATCH 083/110] Updated Fleet init.sls and nginx confs for fleet --- salt/common/nginx/nginx.conf.so-eval | 14 ++++++++++++++ salt/common/nginx/nginx.conf.so-master | 12 ++++++++++++ salt/fleet/init.sls | 12 +++++++++++- 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/salt/common/nginx/nginx.conf.so-eval b/salt/common/nginx/nginx.conf.so-eval index 4ffb2835f..50f48497d 100644 --- a/salt/common/nginx/nginx.conf.so-eval +++ b/salt/common/nginx/nginx.conf.so-eval @@ -110,6 +110,20 @@ http { proxy_set_header Proxy ""; } + + location /api/ { + proxy_pass https://{{ masterip }}:8080/api/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } + location /fleet/ { rewrite /fleet/(.*) /$1 break; proxy_pass https://{{ masterip }}:8080/; diff --git a/salt/common/nginx/nginx.conf.so-master b/salt/common/nginx/nginx.conf.so-master index 535009c71..50f48497d 100644 --- a/salt/common/nginx/nginx.conf.so-master +++ b/salt/common/nginx/nginx.conf.so-master @@ -111,6 +111,18 @@ http { } + location /api/ { + proxy_pass https://{{ masterip }}:8080/api/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } location /fleet/ { rewrite /fleet/(.*) /$1 break; diff --git a/salt/fleet/init.sls b/salt/fleet/init.sls index a90377b1f..c5d77a7ec 100644 --- a/salt/fleet/init.sls +++ b/salt/fleet/init.sls @@ -10,6 +10,13 @@ fleetcdir: - group: 939 - makedirs: True +fleetlogdir: + file.directory: + - name: /opt/so/log/fleet + - user: 939 + - group: 939 + - makedirs: True + fleetdb: mysql_database.present: - name: fleet @@ -44,8 +51,11 @@ so-fleet: - KOLIDE_SERVER_KEY=/ssl/server.key - KOLIDE_LOGGING_JSON=true - KOLIDE_AUTH_JWT_KEY=thisisatest + - KOLIDE_OSQUERY_STATUS_LOG_FILE=/var/log/osquery/status.log + - KOLIDE_OSQUERY_RESULT_LOG_FILE=/var/log/osquery/result.log - binds: - /etc/pki/fleet.key:/ssl/server.key:ro - - /etc/pki/fleet.crt:/ssl/server.cert + - /etc/pki/fleet.crt:/ssl/server.cert:ro + - /opt/so/log/fleet:/var/log/osquery - watch: - /opt/so/conf/fleet/etc From d13e7559fe9349de27edd15b1fc5b588d2a038f3 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 13 Dec 2018 17:32:03 +0000 Subject: [PATCH 084/110] Filebeat - Enabled for master and only enable Bro/Suri inputs when needed --- salt/filebeat/etc/filebeat.yml | 4 ++-- salt/filebeat/init.sls | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 4384d124e..b7ab91e12 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -12,6 +12,7 @@ filebeat.modules: # List of prospectors to fetch data. filebeat.prospectors: #------------------------------ Log prospector -------------------------------- +{%- if grains['role'] == 'so-sensor' or grains['role'] == "so-eval" %} {%- if BROVER != 'SURICATA' %} {%- for LOGNAME in salt['pillar.get']('brologs:enabled', '') %} - type: log @@ -36,6 +37,7 @@ filebeat.prospectors: fields_under_root: true clean_removed: false close_removed: false +{%- endif %} {%- if WAZUHENABLED == '1' %} @@ -73,7 +75,6 @@ output.logstash: # Set gzip compression level. compression_level: 3 - # Enable SSL support. SSL is automatically enabled, if any SSL setting is set. ssl.enabled: true @@ -97,7 +98,6 @@ output.logstash: # Client Certificate Key ssl.key: "/usr/share/filebeat/filebeat.key" - # Elasticsearch template settings #setup.template.settings: diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index da8f0637c..d3a1dfb14 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -63,8 +63,13 @@ so-filebeat: - /opt/so/log/suricata:/suricata:ro - /opt/so/wazuh/logs/alerts/:/wazuh/alerts:ro - /opt/so/wazuh/logs/archives/:/wazuh/archives:ro +{%- if grains['role'] == 'so-master' %} + - /etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro + - /etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro +{%- else %} - /opt/so/conf/filebeat/etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro - /opt/so/conf/filebeat/etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro +{%- endif %} - /etc/ssl/certs/intca.crt:/usr/share/filebeat/intraca.crt:ro - watch: - file: /opt/so/conf/filebeat/etc From 62067f37cfa101f37776cb5cfb6022d89b487663 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 13 Dec 2018 17:33:12 +0000 Subject: [PATCH 085/110] Wazuh - Fix agent ip for storage nodes --- salt/wazuh/files/agent/wazuh-register-agent | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/wazuh/files/agent/wazuh-register-agent b/salt/wazuh/files/agent/wazuh-register-agent index b6199cf9a..7e8574613 100755 --- a/salt/wazuh/files/agent/wazuh-register-agent +++ b/salt/wazuh/files/agent/wazuh-register-agent @@ -1,4 +1,8 @@ +{%- if grains['role'] == 'so-master' or grains['role'] == 'so-eval' %} {%- set ip = salt['pillar.get']('static:masterip', '') %} +{%- elif grains['role'] == 'so-node' } +{% set ip = salt['pillar.get']('node:mainip', '') %} +{%- endif %} #!/bin/bash ### From 5e23859557e1418c6c4ed8c434a9278ef7736aa7 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 13 Dec 2018 17:34:19 +0000 Subject: [PATCH 086/110] Salt - Add Wazuh to other roles --- salt/top.sls | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/top.sls b/salt/top.sls index 03c220047..a319209ca 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -10,6 +10,7 @@ base: {%- if BROVER != 'SURICATA' %} - bro {%- endif %} + - wazuh - filebeat - schedule @@ -49,6 +50,8 @@ base: - logstash - kibana - elastalert + - wazuh + - filebeat - utility - schedule - fleet @@ -87,6 +90,8 @@ base: - logstash - elasticsearch - curator + - wazuh + - filebeat - schedule 'G@role:mastersensor': From 4db52ec8654b115e798db7ca498c32a098f76424 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 13 Dec 2018 17:56:51 +0000 Subject: [PATCH 087/110] Wazuh - Add logic for sensors --- salt/wazuh/files/agent/ossec.conf | 4 ++++ salt/wazuh/files/agent/wazuh-register-agent | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/wazuh/files/agent/ossec.conf b/salt/wazuh/files/agent/ossec.conf index c89b9ce06..b4725075b 100644 --- a/salt/wazuh/files/agent/ossec.conf +++ b/salt/wazuh/files/agent/ossec.conf @@ -1,5 +1,9 @@ {%- if grains['role'] == 'so-master' or grains['role'] == 'so-eval' %} {%- set ip = salt['pillar.get']('static:masterip', '') %} +{%- elif grains['role'] == 'so-node' } +{%- set ip = salt['pillar.get']('node:mainip', '') %} +{%- elif grains['role'] == 'so-sensor' } +{%- set ip = salt['pillar.get']('sensor:mainip', '') %} {%- endif %}