diff --git a/salt/elasticsearch/files/curator/action/close.yml b/salt/curator/files/action/close.yml similarity index 65% rename from salt/elasticsearch/files/curator/action/close.yml rename to salt/curator/files/action/close.yml index a9ca54335..ab4d07050 100644 --- a/salt/elasticsearch/files/curator/action/close.yml +++ b/salt/curator/files/action/close.yml @@ -1,3 +1,13 @@ +{% if grains['role'] == 'so-node' %} + +{%- set cur_close_days = salt['pillar.get']('node:cur_close_days', '') -%} + +{% elif grains['role'] == 'so-eval' %} + +{%- set cur_close_days = salt['pillar.get']('master:cur_close_days', '') -%} + +{%- endif %} + --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" @@ -9,7 +19,7 @@ actions: 1: action: close description: >- - Close indices older than 2 days (based on index name), for logstash- + Close indices older than {{cur_close_days}} days (based on index name), for logstash- prefixed indices. options: delete_aliases: False @@ -26,5 +36,5 @@ actions: direction: older timestring: '%Y.%m.%d' unit: days - unit_count: 2 + unit_count: {{cur_close_days}} exclude: diff --git a/salt/elasticsearch/files/curator/action/delete.yml b/salt/curator/files/action/delete.yml similarity index 62% rename from salt/elasticsearch/files/curator/action/delete.yml rename to salt/curator/files/action/delete.yml index cb55ec0c3..1b3440f8b 100644 --- a/salt/elasticsearch/files/curator/action/delete.yml +++ b/salt/curator/files/action/delete.yml @@ -1,3 +1,12 @@ +{% if grains['role'] == 'so-node' %} + +{%- set log_size_limit = salt['pillar.get']('node:log_size_limit', '') -%} + +{% elif grains['role'] == 'so-eval' %} + +{%- set log_size_limit = salt['pillar.get']('master:log_size_limit', '') -%} + +{%- endif %} --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" @@ -9,7 +18,7 @@ actions: 1: action: delete_indices description: >- - Delete indices when $disk_space value (in GB) is exceeded. + Delete indices when {{log_size_limit}}(GB) is exceeded. options: ignore_empty_list: True disable_action: False @@ -20,4 +29,4 @@ actions: - filtertype: space source: creation_date use_age: True - disk_space: 43 + disk_space: {{log_size_limit}} diff --git a/salt/curator/files/bin/so-curator-close b/salt/curator/files/bin/so-curator-close new file mode 100644 index 000000000..dff6bbb39 --- /dev/null +++ b/salt/curator/files/bin/so-curator-close @@ -0,0 +1,2 @@ +#!/bin/bash +/usr/sbin/so-curator-closed-delete > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/close.yml > /dev/null 2>&1 diff --git a/salt/curator/files/bin/so-curator-closed-delete b/salt/curator/files/bin/so-curator-closed-delete new file mode 100755 index 000000000..4382a721d --- /dev/null +++ b/salt/curator/files/bin/so-curator-closed-delete @@ -0,0 +1,41 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +#. /usr/sbin/so-elastic-common +#. /etc/nsm/securityonion.conf + +# If logrotate script doesn't already exist, create it +#FILE="/etc/logrotate.d/so-curator-closed-delete" +#if ! [ -f ${FILE} ]; then +# cat << EOF > ${FILE} +#/var/log/nsm/so-curator-closed-delete.log { +# daily +# rotate 7 +# copytruncate +# compress +# missingok +# notifempty +#} +#EOF +#fi + +# Avoid starting multiple instances +if pgrep -f "so-curator-closed-delete-delete" >/dev/null; then + echo "Script is already running." +else + /usr/sbin/so-curator-closed-delete-delete +fi diff --git a/salt/curator/files/bin/so-curator-closed-delete-delete b/salt/curator/files/bin/so-curator-closed-delete-delete new file mode 100755 index 000000000..8841c843f --- /dev/null +++ b/salt/curator/files/bin/so-curator-closed-delete-delete @@ -0,0 +1,58 @@ + +{% if grains['role'] == 'so-node' %} + +{%- set ELASTICSEARCH_HOST = salt['pillar.get']('node:mainip', '') -%} +{%- set ELASTICSEARCH_PORT = salt['pillar.get']('node:es_port', '') -%} +{%- set LOG_SIZE_LIMIT = salt['pillar.get']('node:log_size_limit', '') -%} + +{% elif grains['role'] == 'so-eval' %} + +{%- set ELASTICSEARCH_HOST = salt['pillar.get']('master:mainip', '') -%} +{%- set ELASTICSEARCH_PORT = salt['pillar.get']('master:es_port', '') -%} +{%- set LOG_SIZE_LIMIT = salt['pillar.get']('master:log_size_limit', '') -%} + +{%- endif %} + +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +#. /usr/sbin/so-elastic-common +#. /etc/nsm/securityonion.conf + +LOG="/opt/so/log/curator/so-curator-closed-delete.log" + +# Check for 2 conditions: +# 1. Are Elasticsearch indices using more disk space than LOG_SIZE_LIMIT? +# 2. Are there any closed logstash- indices that we can delete? +# If both conditions are true, keep on looping until one of the conditions is false. +while [[ $(du -hs --block-size=1GB /nsm/elasticsearch/nodes | awk '{print $1}' ) -gt "{{LOG_SIZE_LIMIT}}" ]] && +curl -s {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices | grep "^ close logstash-" > /dev/null; do + + # We need to determine OLDEST_INDEX. + # First, get the list of closed indices that are prefixed with "logstash-". + # For example: logstash-ids-YYYY.MM.DD + # Then, sort by date by telling sort to use hyphen as delimiter and then sort on the third field. + # Finally, select the first entry in that sorted list. + OLDEST_INDEX=$(curl -s {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices | grep "^ close logstash-" | awk '{print $2}' | sort -t- -k3 | head -1) + + # Now that we've determined OLDEST_INDEX, ask Elasticsearch to delete it. + curl -XDELETE {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/${OLDEST_INDEX} + + # Finally, write a log entry that says we deleted it. + echo "$(date) - Used disk space exceeds LOG_SIZE_LIMIT ({{LOG_SIZE_LIMIT}} GB) - Index ${OLDEST_INDEX} deleted ..." >> ${LOG} + +done diff --git a/salt/curator/files/bin/so-curator-delete b/salt/curator/files/bin/so-curator-delete new file mode 100644 index 000000000..166497855 --- /dev/null +++ b/salt/curator/files/bin/so-curator-delete @@ -0,0 +1,2 @@ +#!/bin/bash +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/delete.yml > /dev/null 2>&1 diff --git a/salt/elasticsearch/files/curator/curator.yml b/salt/curator/files/curator.yml similarity index 62% rename from salt/elasticsearch/files/curator/curator.yml rename to salt/curator/files/curator.yml index dc8b69309..aa55836b1 100644 --- a/salt/elasticsearch/files/curator/curator.yml +++ b/salt/curator/files/curator.yml @@ -1,9 +1,19 @@ +{% if grains['role'] == 'so-node' %} + +{%- set elasticsearch = salt['pillar.get']('node:mainip', '') -%} + +{% elif grains['role'] == 'so-eval' %} + +{%- set elasticsearch = salt['pillar.get']('master:mainip', '') -%} + +{%- endif %} + --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" client: hosts: - - elasticsearch + - {{elasticsearch}} port: 9200 url_prefix: use_ssl: False diff --git a/salt/curator/init.sls b/salt/curator/init.sls new file mode 100644 index 000000000..adbf4f6b0 --- /dev/null +++ b/salt/curator/init.sls @@ -0,0 +1,136 @@ +{% if grains['role'] == 'so-node' or grains['role'] == 'so-eval' %} +# Curator +# Create the group +curatorgroup: + group.present: + - name: curator + - gid: 934 + +# Add user +curator: + user.present: + - uid: 934 + - gid: 934 + - home: /opt/so/conf/curator + - createhome: False + +# Create the log directory +curactiondir: + file.directory: + - name: /opt/so/conf/curator/action + - user: 934 + - group: 939 + - makedirs: True + +curlogdir: + file.directory: + - name: /opt/so/log/curator + - user: 934 + - group: 939 + +curcloseconf: + file.managed: + - name: /opt/so/conf/curator/action/close.yml + - source: salt://curator/files/action/close.yml + - user: 934 + - group: 939 + - template: jinja + +curdelconf: + file.managed: + - name: /opt/so/conf/curator/action/delete.yml + - source: salt://curator/files/action/delete.yml + - user: 934 + - group: 939 + - template: jinja + +curconf: + file.managed: + - name: /opt/so/conf/curator/curator.yml + - source: salt://curator/files/curator.yml + - user: 934 + - group: 939 + - template: jinja + +curcloseddel: + file.managed: + - name: /usr/sbin/so-curator-closed-delete + - source: salt://curator/files/bin/so-curator-closed-delete + - user: 934 + - group: 939 + - mode: 755 + +curcloseddeldel: + file.managed: + - name: /usr/sbin/so-curator-closed-delete-delete + - source: salt://curator/files/bin/so-curator-closed-delete-delete + - user: 934 + - group: 939 + - mode: 755 + - template: jinja + +curclose: + file.managed: + - name: /usr/sbin/so-curator-close + - source: salt://curator/files/bin/so-curator-close + - user: 934 + - group: 939 + - mode: 755 + +curdel: + file.managed: + - name: /usr/sbin/so-curator-delete + - source: salt://curator/files/bin/so-curator-delete + - user: 934 + - group: 939 + - mode: 755 + +/usr/sbin/so-curator-closed-delete: + cron.present: + - user: root + - minute: '*' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' + +/usr/sbin/so-curator-close: + cron.present: + - user: root + - minute: '*' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' + +/usr/sbin/so-curator-delete: + cron.present: + - user: root + - minute: '*' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' + + +so-curator: + docker_container.running: + - image: soshybridhunter/so-curator:HH1.0.3 + - hostname: curator + - name: so-curator + - user: curator + - interactive: True + - tty: True + - binds: + - /opt/so/conf/curator/curator.yml:/etc/curator/config/curator.yml:ro + - /opt/so/conf/curator/action/:/etc/curator/action:ro + - /opt/so/log/curator:/var/log/curator:rw +# Begin Curator Cron Jobs + +# Close +# Delete +# Hot Warm +# Segment Merge + +# End Curator Cron Jobs +{% endif %} diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 75cc7e5f7..122e2f7eb 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -150,6 +150,7 @@ so-freq: docker_container.running: - image: soshybridhunter/so-freqserver:HH1.0.3 - hostname: freqserver + - name: so-freqserver - user: freqserver - binds: - /opt/so/log/freq_server:/var/log/freq_server:rw @@ -185,7 +186,7 @@ so-domainstats: docker_container.running: - image: soshybridhunter/so-domainstats:HH1.0.3 - hostname: domainstats - - name: domainstats + - name: so-domainstats - user: domainstats - binds: - /opt/so/log/domainstats:/var/log/domain_stats @@ -193,82 +194,6 @@ so-domainstats: {% endif %} -# Curator -# Create the group -curatorgroup: - group.present: - - name: curator - - gid: 934 - -# Add user -curator: - user.present: - - uid: 934 - - gid: 934 - - home: /opt/so/conf/curator - - createhome: False - -# Create the log directory -curactiondir: - file.directory: - - name: /opt/so/conf/curator/action - - user: 934 - - group: 939 - - makedirs: True - -curlogdir: - file.directory: - - name: /opt/so/log/curator - - user: 934 - - group: 939 - -curclose: - file.managed: - - name: /opt/so/conf/curator/action/close.yml - - source: salt://elasticsearch/files/curator/action/close.yml - - user: 934 - - group: 939 - - template: jinja - -curdel: - file.managed: - - name: /opt/so/conf/curator/action/delete.yml - - source: salt://elasticsearch/files/curator/action/delete.yml - - user: 934 - - group: 939 - - template: jinja - -curconf: - file.managed: - - name: /opt/so/conf/curator/curator.yml - - source: salt://elasticsearch/files/curator/curator.yml - - user: 934 - - group: 939 - - template: jinja - -so-curator: - docker_container.running: - - image: soshybridhunter/so-curator:HH1.0.3 - - hostname: curator - - name: curator - - user: curator - - interactive: True - - tty: True - - binds: - - /opt/so/conf/curator/curator.yml:/etc/curator/config/curator.yml:ro - - /opt/so/conf/curator/action/:/etc/curator/action:ro - - /opt/so/log/curator:/var/log/curator:rw - - -# Begin Curator Cron Jobs - -# Close -# Delete -# Hot Warm -# Segment Merge - -# End Curator Cron Jobs - # Elastalert {% if esalert == 1 %} @@ -311,7 +236,7 @@ so-elastalert: docker_container.running: - image: soshybridhunter/so-elastalert:HH1.0.3 - hostname: elastalert - - name: elastalert + - name: so-elastalert - user: elastalert - detach: True - binds: diff --git a/salt/top.sls b/salt/top.sls index 1ab76733e..0b2c50ca4 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -27,8 +27,9 @@ base: - pcap - suricata - bro + - curator - utility - - schedule + - schedule 'G@role:so-master': @@ -60,6 +61,7 @@ base: - firewall - logstash - elasticsearch + - curator - schedule 'G@role:so-node and I@node:node_type:warm': @@ -77,6 +79,7 @@ base: - firewall - logstash - elasticsearch + - curator - schedule 'G@role:mastersensor': diff --git a/so-setup-network.sh b/so-setup-network.sh index 2d013c813..13182a6f7 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -22,6 +22,7 @@ NICS=$(ip link | awk -F: '$0 !~ "lo|vir|veth|br|docker|wl|^[^0-9]"{print $2 " \" CPUCORES=$(cat /proc/cpuinfo | grep processor | wc -l) LISTCORES=$(cat /proc/cpuinfo | grep processor | awk '{print $3 " \"" "core" "\""}') RANDOMUID=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 16 | head -n 1) +NODE_ES_PORT="9200" # End Global Variable Section @@ -397,6 +398,16 @@ get_filesystem_nsm(){ FSNSM=$(df /nsm | awk '$3 ~ /[0-9]+/ { print $2 * 1000 }') } +get_log_size_limit() { + + DISK_SIZE_K=`df /nsm |grep -v "^Filesystem" | awk '{print $2}'` + PERCENTAGE=85 + DISK_SIZE=DISK_SIZE_K*1000 + PERCENTAGE_DISK_SPACE=`echo $(($DISK_SIZE*$PERCENTAGE/100))` + LOG_SIZE_LIMIT=$(($PERCENTAGE_DISK_SPACE/1000000000)) + +} + get_filesystem_root(){ FSROOT=$(df / | awk '$3 ~ /[0-9]+/ { print $2 * 1000 }') } @@ -493,6 +504,9 @@ master_pillar() { echo " oinkcode: $OINKCODE" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls #echo " access_key: $ACCESS_KEY" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls #echo " access_secret: $ACCESS_SECRET" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls + echo " es_port: $NODE_ES_PORT" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls + echo " log_size_limit: $LOG_SIZE_LIMIT" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls + echo " cur_close_days: $CURCLOSEDAYS" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls } @@ -540,6 +554,9 @@ node_pillar() { echo " ls_batch_count: $LSINPUTBATCHCOUNT" >> $TMP/$HOSTNAME.sls echo " es_shard_count: $SHARDCOUNT" >> $TMP/$HOSTNAME.sls echo " node_type: $NODETYPE" >> $TMP/$HOSTNAME.sls + echo " es_port: $NODE_ES_PORT" >> $TMP/$HOSTNAME.sls + echo " log_size_limit: $LOG_SIZE_LIMIT" >> $TMP/$HOSTNAME.sls + echo " cur_close_days: $CURCLOSEDAYS" >> $TMP/$HOSTNAME.sls } @@ -932,6 +949,16 @@ whiptail_check_exitstatus() { } +whiptail_cur_close_days() { + + CURCLOSEDAYS=$(whiptail --title "Security Onion Setup" --inputbox \ + "Please specify the threshold (in days) at which Elasticsearch indices will be closed" 10 60 $CURCLOSEDAYS 3>&1 1>&2 2>&3) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + +} + whiptail_homenet_master() { # Ask for the HOME_NET on the master @@ -979,6 +1006,18 @@ whiptail_install_type() { } +whiptail_log_size_limit() { + + LOG_SIZE_LIMIT=$(whiptail --title "Security Onion Setup" --inputbox \ + "Please specify the amount of disk space (in GB) you would like to allocate for Elasticsearch data storage. \ + By default, this is set to 85% of the disk space allotted for /nsm." 10 60 $LOG_SIZE_LIMIT 3>&1 1>&2 2>&3) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + +} + + whiptail_management_nic() { MNIC=$(whiptail --title "NIC Setup" --radiolist "Please select your management NIC" 20 78 12 ${NICS[@]} 3>&1 1>&2 2>&3 ) @@ -1508,11 +1547,13 @@ if (whiptail_you_sure); then NSMSETUP=BASIC NIDS=Suricata BROVERSION=ZEEK + CURCLOSEDAYS=30 whiptail_make_changes clear_master mkdir -p /nsm get_filesystem_root get_filesystem_nsm + get_log_size_limit get_main_ip # Add the user so we can sit back and relax echo "" @@ -1553,6 +1594,10 @@ if (whiptail_you_sure); then whiptail_management_server whiptail_master_updates set_updates + get_log_size_limit + whiptail_log_size_limit + CURCLOSEDAYS=30 + whiptail_cur_close_days es_heapsize ls_heapsize whiptail_node_advanced