Curator - break out from ES and add config

salt/curator/files/action/close.yml

@@ -0,0 +1,34 @@
+{% if grains['role'] == 'so-node' or grains['role'] == 'so-eval' %}
+{%- set cur_close_days = salt['pillar.get']('node:cur_close_days', '') -%}
+{%- endif %}
+
+---
+# Remember, leave a key empty if there is no value.  None will be a string,
+# not a Python "NoneType"
+#
+# Also remember that all examples have 'disable_action' set to True.  If you
+# want to use this action as a template, be sure to set this to False after
+# copying it.
+actions:
+  1:
+    action: close
+    description: >-
+      Close indices older than {{cur_close_days}} days (based on index name), for logstash-
+      prefixed indices.
+    options:
+      delete_aliases: False
+      timeout_override:
+      continue_if_exception: False
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: prefix
+      value: logstash-
+      exclude:
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: {{cur_close_days}}
+      exclude:

salt/curator/files/action/delete.yml

@@ -0,0 +1,26 @@
+{% if grains['role'] == 'so-node' or grains['role'] == 'so-eval' %}
+{%- set log_size_limit = salt['pillar.get']('node:log_size_limit', '') -%}
+{%- endif %}
+---
+# Remember, leave a key empty if there is no value.  None will be a string,
+# not a Python "NoneType"
+#
+# Also remember that all examples have 'disable_action' set to True.  If you
+# want to use this action as a template, be sure to set this to False after
+# copying it.
+actions:
+  1:
+    action: delete_indices
+    description: >-
+      Delete indices when {{log_size_limit}}(GB) is exceeded.
+    options:
+      ignore_empty_list: True
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: prefix
+      value: logstash-
+    - filtertype: space
+      source: creation_date
+      use_age: True
+      disk_space: {{log_size_limit}}

salt/curator/files/bin/so-curator-close

@@ -0,0 +1,2 @@
+#!/bin/bash
+/usr/sbin/so-curator-closed-delete > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/close.yml > /dev/null 2>&1

salt/curator/files/bin/so-curator-closed-delete

@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#. /usr/sbin/so-elastic-common
+#. /etc/nsm/securityonion.conf
+
+# If logrotate script doesn't already exist, create it
+#FILE="/etc/logrotate.d/so-curator-closed-delete"
+#if ! [ -f ${FILE} ]; then
+#        cat << EOF > ${FILE}
+#/var/log/nsm/so-curator-closed-delete.log {
+#    daily
+#    rotate 7
+#    copytruncate
+#    compress
+#    missingok
+#    notifempty
+#}
+#EOF
+#fi
+
+# Avoid starting multiple instances
+if pgrep -f "so-curator-closed-delete-delete" >/dev/null; then
+	echo "Script is already running."
+else
+	/usr/local/bin/so-curator-closed-delete-delete
+fi

salt/curator/files/bin/so-curator-closed-delete-delete

@@ -0,0 +1,50 @@
+
+{% if grains['role'] == 'so-node' or grains['role'] == 'so-eval' %}
+{%- set ELASTICSEARCH_HOST = salt['pillar.get']('node:mainip', '') -%}
+{%- set ELASTICSEARCH_PORT = salt['pillar.get']('node:es_port', '') -%}
+{%- set LOG_SIZE_LIMIT = salt['pillar.get']('node:log_size_limit', '') -%}
+{%- endif %}
+
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#. /usr/sbin/so-elastic-common
+#. /etc/nsm/securityonion.conf
+
+LOG="/opt/so/log/curator/so-curator-closed-delete.log"
+
+# Check for 2 conditions:
+# 1. Are Elasticsearch indices using more disk space than LOG_SIZE_LIMIT?
+# 2. Are there any closed logstash- indices that we can delete?
+# If both conditions are true, keep on looping until one of the conditions is false.
+while [[ $(du -hs --block-size=1GB /nsm/elasticsearch/nodes | awk '{print $1}' ) -gt "{{LOG_SIZE_LIMIT}}" ]] &&
+curl -s {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices | grep "^      close logstash-" > /dev/null; do
+
+	# We need to determine OLDEST_INDEX.
+	# First, get the list of closed indices that are prefixed with "logstash-".
+	# For example: logstash-ids-YYYY.MM.DD
+	# Then, sort by date by telling sort to use hyphen as delimiter and then sort on the third field.
+	# Finally, select the first entry in that sorted list.
+	OLDEST_INDEX=$(curl -s {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices | grep "^      close logstash-" | awk '{print $2}' | sort -t- -k3 | head -1)
+
+	# Now that we've determined OLDEST_INDEX, ask Elasticsearch to delete it.
+	curl -XDELETE {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/${OLDEST_INDEX}
+
+	# Finally, write a log entry that says we deleted it.
+	echo "$(date) - Used disk space exceeds LOG_SIZE_LIMIT ({{LOG_SIZE_LIMIT}} GB) - Index ${OLDEST_INDEX} deleted ..." >> ${LOG}
+
+done

salt/curator/files/bin/so-curator-delete

@@ -0,0 +1,2 @@
+#!/bin/bash
+docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/delete.yml > /dev/null 2>&1

salt/curator/files/curator.yml

@@ -0,0 +1,26 @@
+{% if grains['role'] == 'so-node' or grains['role'] == 'so-eval' %}
+{%- set elasticsearch = salt['pillar.get']('node:mainip', '') -%}
+{%- endif %}
+
+---
+# Remember, leave a key empty if there is no value.  None will be a string,
+# not a Python "NoneType"
+client:
+  hosts:
+    - {{elasticsearch}}
+  port: 9200
+  url_prefix:
+  use_ssl: False
+  certificate:
+  client_cert:
+  client_key:
+  ssl_no_validate: False
+  http_auth:
+  timeout: 30
+  master_only: False
+
+logging:
+  loglevel: INFO
+  logfile: '/var/log/curator/curator.log'
+  logformat: default
+  blacklist: ['elasticsearch', 'urllib3']

salt/curator/init.sls

@@ -0,0 +1,136 @@
+{% if grains['role'] == 'so-node' or grains['role'] == 'so-eval' %}
+# Curator
+# Create the group
+curatorgroup:
+  group.present:
+    - name: curator
+    - gid: 934
+
+# Add user
+curator:
+  user.present:
+    - uid: 934
+    - gid: 934
+    - home: /opt/so/conf/curator
+    - createhome: False
+
+# Create the log directory
+curactiondir:
+  file.directory:
+    - name: /opt/so/conf/curator/action
+    - user: 934
+    - group: 939
+    - makedirs: True
+
+curlogdir:
+  file.directory:
+    - name: /opt/so/log/curator
+    - user: 934
+    - group: 939
+
+curcloseconf:
+  file.managed:
+    - name: /opt/so/conf/curator/action/close.yml
+    - source: salt://curator/files/action/close.yml
+    - user: 934
+    - group: 939
+    - template: jinja
+
+curdelconf:
+  file.managed:
+    - name: /opt/so/conf/curator/action/delete.yml
+    - source: salt://curator/files/action/delete.yml
+    - user: 934
+    - group: 939
+    - template: jinja
+
+curconf:
+  file.managed:
+    - name: /opt/so/conf/curator/curator.yml
+    - source: salt://curator/files/curator.yml
+    - user: 934
+    - group: 939
+    - template: jinja
+
+curcloseddel:
+  file.managed:
+    - name: /usr/local/bin/so-curator-closed-delete
+    - source: salt://curator/files/bin/so-curator-closed-delete
+    - user: 934
+    - group: 939
+    - mode: 755
+
+curcloseddeldel:
+  file.managed:
+    - name: /usr/local/bin/so-curator-closed-delete-delete
+    - source: salt://curator/files/bin/so-curator-closed-delete-delete
+    - user: 934
+    - group: 939
+    - mode: 755
+    - template: jinja
+
+curclose:
+  file.managed:
+    - name: /usr/local/bin/so-curator-close
+    - source: salt://curator/files/bin/so-curator-close
+    - user: 934
+    - group: 939
+    - mode: 755
+
+curdel:
+  file.managed:
+    - name: /usr/local/bin/so-curator-delete
+    - source: salt://curator/files/bin/so-curator-delete
+    - user: 934
+    - group: 939
+    - mode: 755
+
+/usr/local/bin/so-curator-closed-delete:
+ cron.present:
+   - user: root
+   - minute: '*'
+   - hour: '*'
+   - daymonth: '*'
+   - month: '*'
+   - dayweek: '*'
+
+/usr/local/bin/so-curator-close:
+ cron.present:
+   - user: root
+   - minute: '*'
+   - hour: '*'
+   - daymonth: '*'
+   - month: '*'
+   - dayweek: '*'
+
+/usr/local/bin/so-curator-delete:
+ cron.present:
+   - user: root
+   - minute: '*'
+   - hour: '*'
+   - daymonth: '*'
+   - month: '*'
+   - dayweek: '*'
+
+
+so-curator:
+  docker_container.running:
+    - image: soshybridhunter/so-curator:HH1.0.3
+    - hostname: curator
+    - name: so-curator
+    - user: curator
+    - interactive: True
+    - tty: True
+    - binds:
+      - /opt/so/conf/curator/curator.yml:/etc/curator/config/curator.yml:ro
+      - /opt/so/conf/curator/action/:/etc/curator/action:ro
+      - /opt/so/log/curator:/var/log/curator:rw
+# Begin Curator Cron Jobs
+
+# Close
+# Delete
+# Hot Warm
+# Segment Merge
+
+# End Curator Cron Jobs
+{% endif %}