From 19cfce5e0bc83e41a1adbbaa313876e34df03d5e Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 23 Aug 2021 10:47:41 -0400 Subject: [PATCH 01/61] Add curator delete yml files --- salt/curator/files/action/so-beats-delete.yml | 29 +++++++++++++++++++ .../files/action/so-firewall-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-ids-delete.yml | 29 +++++++++++++++++++ .../curator/files/action/so-import-delete.yml | 29 +++++++++++++++++++ .../files/action/so-osquery-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-ossec-delete.yml | 29 +++++++++++++++++++ .../files/action/so-strelka-delete.yml | 29 +++++++++++++++++++ .../curator/files/action/so-syslog-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-zeek-delete.yml | 29 +++++++++++++++++++ 9 files changed, 261 insertions(+) create mode 100644 salt/curator/files/action/so-beats-delete.yml create mode 100644 salt/curator/files/action/so-firewall-delete.yml create mode 100644 salt/curator/files/action/so-ids-delete.yml create mode 100644 salt/curator/files/action/so-import-delete.yml create mode 100644 salt/curator/files/action/so-osquery-delete.yml create mode 100644 salt/curator/files/action/so-ossec-delete.yml create mode 100644 salt/curator/files/action/so-strelka-delete.yml create mode 100644 salt/curator/files/action/so-syslog-delete.yml create mode 100644 salt/curator/files/action/so-zeek-delete.yml diff --git a/salt/curator/files/action/so-beats-delete.yml b/salt/curator/files/action/so-beats-delete.yml new file mode 100644 index 000000000..77931d661 --- /dev/null +++ b/salt/curator/files/action/so-beats-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-beats:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete beats indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-beats.*|so-beats.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-firewall-delete.yml b/salt/curator/files/action/so-firewall-delete.yml new file mode 100644 index 000000000..7588de437 --- /dev/null +++ b/salt/curator/files/action/so-firewall-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-firewall:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete firewall indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-firewall.*|so-firewall.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-ids-delete.yml b/salt/curator/files/action/so-ids-delete.yml new file mode 100644 index 000000000..e5bda4e34 --- /dev/null +++ b/salt/curator/files/action/so-ids-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-ids:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete IDS indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-ids.*|so-ids.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-import-delete.yml b/salt/curator/files/action/so-import-delete.yml new file mode 100644 index 000000000..aa9808c5f --- /dev/null +++ b/salt/curator/files/action/so-import-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-import:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete import indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-import.*|so-import.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-osquery-delete.yml b/salt/curator/files/action/so-osquery-delete.yml new file mode 100644 index 000000000..d77b1b3d1 --- /dev/null +++ b/salt/curator/files/action/so-osquery-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-osquery:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete import indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-osquery.*|so-osquery.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-ossec-delete.yml b/salt/curator/files/action/so-ossec-delete.yml new file mode 100644 index 000000000..7aea13e41 --- /dev/null +++ b/salt/curator/files/action/so-ossec-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-ossec:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete ossec indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-ossec.*|so-ossec.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-strelka-delete.yml b/salt/curator/files/action/so-strelka-delete.yml new file mode 100644 index 000000000..3487aeb6d --- /dev/null +++ b/salt/curator/files/action/so-strelka-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-strelka:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete Strelka indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-strelka.*|so-strelka.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-syslog-delete.yml b/salt/curator/files/action/so-syslog-delete.yml new file mode 100644 index 000000000..5fe7417ad --- /dev/null +++ b/salt/curator/files/action/so-syslog-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-syslog:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete syslog indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-syslog.*|so-syslog.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-zeek-delete.yml b/salt/curator/files/action/so-zeek-delete.yml new file mode 100644 index 000000000..0694c2aed --- /dev/null +++ b/salt/curator/files/action/so-zeek-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete Zeek indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-zeek.*|so-zeek.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file From de5f823abff6ac67e4fc6f50cede4ca44da08c84 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 25 Aug 2021 18:28:17 -0400 Subject: [PATCH 02/61] Add automation for deploy-vader env --- .../automation/vader-distributed-iso-manager | 76 ++++++++++++++++++ setup/automation/vader-distributed-iso-search | 78 ++++++++++++++++++ setup/automation/vader-distributed-iso-sensor | 80 +++++++++++++++++++ 3 files changed, 234 insertions(+) create mode 100644 setup/automation/vader-distributed-iso-manager create mode 100644 setup/automation/vader-distributed-iso-search create mode 100644 setup/automation/vader-distributed-iso-sensor diff --git a/setup/automation/vader-distributed-iso-manager b/setup/automation/vader-distributed-iso-manager new file mode 100644 index 000000000..07a22b588 --- /dev/null +++ b/setup/automation/vader-distributed-iso-manager @@ -0,0 +1,76 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +TESTING=true + +address_type=DHCP +ADMINUSER=onionuser +ADMINPASS1=onionuser +ADMINPASS2=onionuser +ALLOW_CIDR=0.0.0.0/0 +ALLOW_ROLE=a +BASICZEEK=7 +BASICSURI=7 +# BLOGS= +#BNICS=eth1 +ZEEKVERSION=ZEEK +# CURCLOSEDAYS= +# EVALADVANCED=BASIC +GRAFANA=1 +# HELIXAPIKEY= +HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 +HNSENSOR=inherit +HOSTNAME=distributed-manager +install_type=MANAGER +# LSINPUTBATCHCOUNT= +# LSINPUTTHREADS= +# LSPIPELINEBATCH= +# LSPIPELINEWORKERS= +MANAGERADV=BASIC +# MDNS= +# MGATEWAY= +# MIP= +# MMASK= +MNIC=eth0 +# MSEARCH= +# MSRV= +# MTU= +NIDS=Suricata +# NODE_ES_HEAP_SIZE= +# NODE_LS_HEAP_SIZE= +NODESETUP=NODEBASIC +NSMSETUP=BASIC +NODEUPDATES=MANAGER +# OINKCODE= +OSQUERY=1 +# PATCHSCHEDULEDAYS= +# PATCHSCHEDULEHOURS= +PATCHSCHEDULENAME=auto +PLAYBOOK=1 +# REDIRECTHOST= +REDIRECTINFO=IP +RULESETUP=ETOPEN +# SHARDCOUNT= +# SKIP_REBOOT= +SOREMOTEPASS1=onionuser +SOREMOTEPASS2=onionuser +STRELKA=1 +THEHIVE=1 +WAZUH=1 +WEBUSER=onionuser@somewhere.invalid +WEBPASSWD1=0n10nus3r +WEBPASSWD2=0n10nus3r diff --git a/setup/automation/vader-distributed-iso-search b/setup/automation/vader-distributed-iso-search new file mode 100644 index 000000000..c1ced1537 --- /dev/null +++ b/setup/automation/vader-distributed-iso-search @@ -0,0 +1,78 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +TESTING=true + +address_type=DHCP +ADMINUSER=onionuser +ADMINPASS1=onionuser +ADMINPASS2=onionuser +# ALLOW_CIDR=0.0.0.0/0 +# ALLOW_ROLE=a +# BASICZEEK=7 +# BASICSURI=7 +# BLOGS= +# BNICS=eth1 +# ZEEKVERSION=ZEEK +# CURCLOSEDAYS= +# EVALADVANCED=BASIC +# GRAFANA=1 +# HELIXAPIKEY= +HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 +HNSENSOR=inherit +HOSTNAME=distributed-search +install_type=SEARCHNODE +# LSINPUTBATCHCOUNT= +# LSINPUTTHREADS= +# LSPIPELINEBATCH= +# LSPIPELINEWORKERS= +# MANAGERADV=BASIC +MANAGERUPDATES=1 +# MDNS= +# MGATEWAY= +# MIP= +# MMASK= +MNIC=eth0 +# MSEARCH= +MSRV=distributed-manager +MSRVIP=10.66.166.100 +# MTU= +# NIDS=Suricata +# NODE_ES_HEAP_SIZE= +# NODE_LS_HEAP_SIZE= +NODESETUP=NODEBASIC +NSMSETUP=ADVANCED +NODEUPDATES=MANAGER +# OINKCODE= +# OSQUERY=1 +# PATCHSCHEDULEDAYS= +# PATCHSCHEDULEHOURS= +PATCHSCHEDULENAME=auto +# PLAYBOOK=1 +# REDIRECTHOST= +# REDIRECTINFO=IP +# RULESETUP=ETOPEN +# SHARDCOUNT= +# SKIP_REBOOT= +SOREMOTEPASS1=onionuser +SOREMOTEPASS2=onionuser +# STRELKA=1 +# THEHIVE=1 +# WAZUH=1 +# WEBUSER=onionuser@somewhere.invalid +# WEBPASSWD1=0n10nus3r +# WEBPASSWD2=0n10nus3r diff --git a/setup/automation/vader-distributed-iso-sensor b/setup/automation/vader-distributed-iso-sensor new file mode 100644 index 000000000..356e67a2d --- /dev/null +++ b/setup/automation/vader-distributed-iso-sensor @@ -0,0 +1,80 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +TESTING=true + +address_type=DHCP +ADMINUSER=onionuser +ADMINPASS1=onionuser +ADMINPASS2=onionuser +# ALLOW_CIDR=0.0.0.0/0 +# ALLOW_ROLE=a +BASICZEEK=2 +BASICSURI=2 +# BLOGS= +BNICS=eth1 +ZEEKVERSION=ZEEK +# CURCLOSEDAYS= +# EVALADVANCED=BASIC +# GRAFANA=1 +# HELIXAPIKEY= +HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 +HNSENSOR=inherit +HOSTNAME=Distributed-Sensor +install_type=SENSOR +# LSINPUTBATCHCOUNT= +# LSINPUTTHREADS= +# LSPIPELINEBATCH= +# LSPIPELINEWORKERS= +# MANAGERADV=BASIC +MANAGERUPDATES=1 +# MDNS= +# MGATEWAY= +# MIP= +# MMASK= +MNIC=eth0 +# MSEARCH= +MSRV=distributed-manager +MSRVIP=10.66.166.100 +MTU=1500 +# NIDS=Suricata +# NODE_ES_HEAP_SIZE= +# NODE_LS_HEAP_SIZE= +# NODESETUP=NODEBASIC +NSMSETUP=ADVANCED +NODEUPDATES=MANAGER +# OINKCODE= +# OSQUERY=1 +# PATCHSCHEDULEDAYS= +# PATCHSCHEDULEHOURS= +PATCHSCHEDULENAME=auto +# PLAYBOOK=1 +# REDIRECTHOST= +# REDIRECTINFO=IP +# RULESETUP=ETOPEN +# SHARDCOUNT= +# SKIP_REBOOT= +SOREMOTEPASS1=onionuser +SOREMOTEPASS2=onionuser +# STRELKA=1 +SURIPINS=(2 3) +# THEHIVE=1 +# WAZUH=1 +# WEBUSER=onionuser@somewhere.invalid +# WEBPASSWD1=0n10nus3r +# WEBPASSWD2=0n10nus3r +ZEEKPINS=(0 1) From 09d50978379ab7ee6a8cf2080cf6532772edb09e Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 25 Aug 2021 21:08:49 -0400 Subject: [PATCH 03/61] Remove unused automation files --- .../automation/vader-distributed-iso-manager | 76 ------------------ setup/automation/vader-distributed-iso-search | 78 ------------------ setup/automation/vader-distributed-iso-sensor | 80 ------------------- 3 files changed, 234 deletions(-) delete mode 100644 setup/automation/vader-distributed-iso-manager delete mode 100644 setup/automation/vader-distributed-iso-search delete mode 100644 setup/automation/vader-distributed-iso-sensor diff --git a/setup/automation/vader-distributed-iso-manager b/setup/automation/vader-distributed-iso-manager deleted file mode 100644 index 07a22b588..000000000 --- a/setup/automation/vader-distributed-iso-manager +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash - -# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -TESTING=true - -address_type=DHCP -ADMINUSER=onionuser -ADMINPASS1=onionuser -ADMINPASS2=onionuser -ALLOW_CIDR=0.0.0.0/0 -ALLOW_ROLE=a -BASICZEEK=7 -BASICSURI=7 -# BLOGS= -#BNICS=eth1 -ZEEKVERSION=ZEEK -# CURCLOSEDAYS= -# EVALADVANCED=BASIC -GRAFANA=1 -# HELIXAPIKEY= -HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 -HNSENSOR=inherit -HOSTNAME=distributed-manager -install_type=MANAGER -# LSINPUTBATCHCOUNT= -# LSINPUTTHREADS= -# LSPIPELINEBATCH= -# LSPIPELINEWORKERS= -MANAGERADV=BASIC -# MDNS= -# MGATEWAY= -# MIP= -# MMASK= -MNIC=eth0 -# MSEARCH= -# MSRV= -# MTU= -NIDS=Suricata -# NODE_ES_HEAP_SIZE= -# NODE_LS_HEAP_SIZE= -NODESETUP=NODEBASIC -NSMSETUP=BASIC -NODEUPDATES=MANAGER -# OINKCODE= -OSQUERY=1 -# PATCHSCHEDULEDAYS= -# PATCHSCHEDULEHOURS= -PATCHSCHEDULENAME=auto -PLAYBOOK=1 -# REDIRECTHOST= -REDIRECTINFO=IP -RULESETUP=ETOPEN -# SHARDCOUNT= -# SKIP_REBOOT= -SOREMOTEPASS1=onionuser -SOREMOTEPASS2=onionuser -STRELKA=1 -THEHIVE=1 -WAZUH=1 -WEBUSER=onionuser@somewhere.invalid -WEBPASSWD1=0n10nus3r -WEBPASSWD2=0n10nus3r diff --git a/setup/automation/vader-distributed-iso-search b/setup/automation/vader-distributed-iso-search deleted file mode 100644 index c1ced1537..000000000 --- a/setup/automation/vader-distributed-iso-search +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash - -# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -TESTING=true - -address_type=DHCP -ADMINUSER=onionuser -ADMINPASS1=onionuser -ADMINPASS2=onionuser -# ALLOW_CIDR=0.0.0.0/0 -# ALLOW_ROLE=a -# BASICZEEK=7 -# BASICSURI=7 -# BLOGS= -# BNICS=eth1 -# ZEEKVERSION=ZEEK -# CURCLOSEDAYS= -# EVALADVANCED=BASIC -# GRAFANA=1 -# HELIXAPIKEY= -HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 -HNSENSOR=inherit -HOSTNAME=distributed-search -install_type=SEARCHNODE -# LSINPUTBATCHCOUNT= -# LSINPUTTHREADS= -# LSPIPELINEBATCH= -# LSPIPELINEWORKERS= -# MANAGERADV=BASIC -MANAGERUPDATES=1 -# MDNS= -# MGATEWAY= -# MIP= -# MMASK= -MNIC=eth0 -# MSEARCH= -MSRV=distributed-manager -MSRVIP=10.66.166.100 -# MTU= -# NIDS=Suricata -# NODE_ES_HEAP_SIZE= -# NODE_LS_HEAP_SIZE= -NODESETUP=NODEBASIC -NSMSETUP=ADVANCED -NODEUPDATES=MANAGER -# OINKCODE= -# OSQUERY=1 -# PATCHSCHEDULEDAYS= -# PATCHSCHEDULEHOURS= -PATCHSCHEDULENAME=auto -# PLAYBOOK=1 -# REDIRECTHOST= -# REDIRECTINFO=IP -# RULESETUP=ETOPEN -# SHARDCOUNT= -# SKIP_REBOOT= -SOREMOTEPASS1=onionuser -SOREMOTEPASS2=onionuser -# STRELKA=1 -# THEHIVE=1 -# WAZUH=1 -# WEBUSER=onionuser@somewhere.invalid -# WEBPASSWD1=0n10nus3r -# WEBPASSWD2=0n10nus3r diff --git a/setup/automation/vader-distributed-iso-sensor b/setup/automation/vader-distributed-iso-sensor deleted file mode 100644 index 356e67a2d..000000000 --- a/setup/automation/vader-distributed-iso-sensor +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash - -# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -TESTING=true - -address_type=DHCP -ADMINUSER=onionuser -ADMINPASS1=onionuser -ADMINPASS2=onionuser -# ALLOW_CIDR=0.0.0.0/0 -# ALLOW_ROLE=a -BASICZEEK=2 -BASICSURI=2 -# BLOGS= -BNICS=eth1 -ZEEKVERSION=ZEEK -# CURCLOSEDAYS= -# EVALADVANCED=BASIC -# GRAFANA=1 -# HELIXAPIKEY= -HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 -HNSENSOR=inherit -HOSTNAME=Distributed-Sensor -install_type=SENSOR -# LSINPUTBATCHCOUNT= -# LSINPUTTHREADS= -# LSPIPELINEBATCH= -# LSPIPELINEWORKERS= -# MANAGERADV=BASIC -MANAGERUPDATES=1 -# MDNS= -# MGATEWAY= -# MIP= -# MMASK= -MNIC=eth0 -# MSEARCH= -MSRV=distributed-manager -MSRVIP=10.66.166.100 -MTU=1500 -# NIDS=Suricata -# NODE_ES_HEAP_SIZE= -# NODE_LS_HEAP_SIZE= -# NODESETUP=NODEBASIC -NSMSETUP=ADVANCED -NODEUPDATES=MANAGER -# OINKCODE= -# OSQUERY=1 -# PATCHSCHEDULEDAYS= -# PATCHSCHEDULEHOURS= -PATCHSCHEDULENAME=auto -# PLAYBOOK=1 -# REDIRECTHOST= -# REDIRECTINFO=IP -# RULESETUP=ETOPEN -# SHARDCOUNT= -# SKIP_REBOOT= -SOREMOTEPASS1=onionuser -SOREMOTEPASS2=onionuser -# STRELKA=1 -SURIPINS=(2 3) -# THEHIVE=1 -# WAZUH=1 -# WEBUSER=onionuser@somewhere.invalid -# WEBPASSWD1=0n10nus3r -# WEBPASSWD2=0n10nus3r -ZEEKPINS=(0 1) From c5d120293df57e96af46195f33d9c4a4f3630c24 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 27 Aug 2021 11:33:51 -0400 Subject: [PATCH 04/61] Initial work to add unattended option to soup --- salt/common/tools/sbin/soup | 91 +++++++++++++++++++++++-------------- 1 file changed, 58 insertions(+), 33 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index ce8923e90..8adabf908 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -105,17 +105,20 @@ add_common() { airgap_mounted() { # Let's see if the ISO is already mounted. - if [ -f /tmp/soagupdate/SecurityOnion/VERSION ]; then + if [[ -f /tmp/soagupdate/SecurityOnion/VERSION ]]; then echo "The ISO is already mounted" else - echo "" - cat << EOF + if [[ -z $ISOLOC ]]; then + echo "This is airgap. Ask for a location." + echo "" + cat << EOF In order for soup to proceed, the path to the downloaded Security Onion ISO file, or the path to the CD-ROM or equivalent device containing the ISO media must be provided. For example, if you have copied the new Security Onion ISO file to your home directory, then the path might look like /home/myuser/securityonion-2.x.y.iso. Or, if you have burned the new ISO onto an optical disk then the path might look like /dev/cdrom. EOF - read -rp 'Enter the path to the new Security Onion ISO content: ' ISOLOC + read -rp 'Enter the path to the new Security Onion ISO content: ' ISOLOC + fi if [[ -f $ISOLOC ]]; then # Mounting the ISO image mkdir -p /tmp/soagupdate @@ -124,23 +127,27 @@ EOF if [ ! -f /tmp/soagupdate/SecurityOnion/VERSION ]; then echo "Something went wrong trying to mount the ISO." echo "Ensure you verify the ISO that you downloaded." - exit 0 + exit 1 else echo "ISO has been mounted!" fi elif [[ -f $ISOLOC/SecurityOnion/VERSION ]]; then ln -s $ISOLOC /tmp/soagupdate echo "Found the update content" - else + elif [[ -b $ISOLOC ]]; then mkdir -p /tmp/soagupdate mount $ISOLOC /tmp/soagupdate if [ ! -f /tmp/soagupdate/SecurityOnion/VERSION ]; then echo "Something went wrong trying to mount the device." echo "Ensure you verify the ISO that you downloaded." - exit 0 + exit 1 else echo "Device has been mounted!" - fi + fi + else + echo "Could not find Security Onion ISO content at ${ISOLOC}" + echo "Ensure the path you entered is correct, and that you verify the ISO that you downloaded." + exit 1 fi fi } @@ -774,39 +781,22 @@ verify_latest_update_script() { } main() { - set -e - set +e trap 'check_err $?' EXIT - echo "### Preparing soup at $(date) ###" - while getopts ":b" opt; do - case "$opt" in - b ) # process option b - shift - BATCHSIZE=$1 - if ! [[ "$BATCHSIZE" =~ ^[0-9]+$ ]]; then - echo "Batch size must be a number greater than 0." - exit 1 - fi - ;; - \? ) - echo "Usage: cmd [-b]" - ;; - esac - done - + echo "Checking to see if this is an airgap install." + echo "" + check_airgap + if [[ $is_airgap -eq 0 && $UNATTENDED == true && -z $ISOLOC ]]; then + echo "Missing file argument for unattended airgap upgrade." + fi echo "Checking to see if this is a manager." echo "" require_manager set_minionid - echo "Checking to see if this is an airgap install." - echo "" - check_airgap echo "Found that Security Onion $INSTALLEDVERSION is currently installed." echo "" if [[ $is_airgap -eq 0 ]]; then # Let's mount the ISO since this is airgap - echo "This is airgap. Ask for a location." airgap_mounted else echo "Cloning Security Onion github repo into $UPDATE_DIR." @@ -1029,7 +1019,40 @@ EOF echo "### soup has been served at $(date) ###" } -cat << EOF +while getopts ":b:f:y" opt; do + case ${opt} in + b ) + BATCHSIZE="$OPTARG" + if ! [[ "$BATCHSIZE" =~ ^[0-9]+$ ]]; then + echo "Batch size must be a number greater than 0." + exit 1 + fi + ;; + y ) + if [[ ! -f /opt/so/state/yeselastic.txt ]]; then + echo "Cannot run soup in unattended mode. You must run soup manually to accept the Elastic License." + exit 1 + else + UNATTENDED=true + fi + ;; + f ) + ISOLOC="$OPTARG" + ;; + \? ) + echo "Usage: soup [-b] [-y] [-f ]" + exit 1 + ;; + : ) + echo "Invalid option: $OPTARG requires an argument" + exit 1 + ;; + esac +done +shift $((OPTIND - 1)) + +if [[ -z $UNATTENDED ]]; then + cat << EOF SOUP - Security Onion UPdater @@ -1041,7 +1064,9 @@ Press Enter to continue or Ctrl-C to cancel. EOF -read -r input + read -r input +fi +echo "### Preparing soup at $(date) ###" main "$@" | tee -a $SOUP_LOG From ca5339341f6f39fc91f6d3e6fb089d4e71c4b5a4 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 27 Aug 2021 11:34:28 -0400 Subject: [PATCH 05/61] Fix batch size regex to disallow 0 --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 8adabf908..60e0ce93f 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -1023,7 +1023,7 @@ while getopts ":b:f:y" opt; do case ${opt} in b ) BATCHSIZE="$OPTARG" - if ! [[ "$BATCHSIZE" =~ ^[0-9]+$ ]]; then + if ! [[ "$BATCHSIZE" =~ ^[1-9][0-9]*$ ]]; then echo "Batch size must be a number greater than 0." exit 1 fi From f8cdf5bca3987112ceb13305a124a5a2ae6a880a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 27 Aug 2021 14:39:02 -0400 Subject: [PATCH 06/61] Add Templates for all filebeat modules --- .../templates/so/so-aws-template.json.jinja | 13 +++++++++++++ .../templates/so/so-azure-template.json.jinja | 13 +++++++++++++ .../templates/so/so-barracuda-template.json.jinja | 13 +++++++++++++ .../templates/so/so-bluecoat-template.json.jinja | 13 +++++++++++++ .../templates/so/so-cef-template.json.jinja | 13 +++++++++++++ .../templates/so/so-checkpoint-template.json.jinja | 13 +++++++++++++ .../templates/so/so-cisco-template.json.jinja | 13 +++++++++++++ .../templates/so/so-cyberark-template.json.jinja | 13 +++++++++++++ .../templates/so/so-cylance-template.json.jinja | 13 +++++++++++++ .../so/so-elasticsearch-template.json.jinja | 13 +++++++++++++ .../templates/so/so-f5-template.json.jinja | 13 +++++++++++++ .../templates/so/so-fortinet-template.json.jinja | 13 +++++++++++++ .../templates/so/so-gcp-template.json.jinja | 13 +++++++++++++ .../so/so-google_workspace-template.json.jinja | 13 +++++++++++++ .../templates/so/so-imperva-template.json.jinja | 13 +++++++++++++ .../templates/so/so-infoblox-template.json.jinja | 13 +++++++++++++ .../templates/so/so-juniper-template.json.jinja | 13 +++++++++++++ .../templates/so/so-kibana-template.json.jinja | 13 +++++++++++++ .../templates/so/so-logstash-template.json.jinja | 13 +++++++++++++ .../templates/so/so-microsoft-template.json.jinja | 13 +++++++++++++ .../templates/so/so-misp-template.json.jinja | 13 +++++++++++++ .../templates/so/so-netflow-template.json.jinja | 13 +++++++++++++ .../templates/so/so-netscout-template.json.jinja | 13 +++++++++++++ .../templates/so/so-o365-template.json.jinja | 13 +++++++++++++ .../templates/so/so-okta-template.json.jinja | 13 +++++++++++++ .../templates/so/so-proofpoint-template.json.jinja | 13 +++++++++++++ .../templates/so/so-radware-template.json.jinja | 13 +++++++++++++ .../templates/so/so-redis-template.json.jinja | 13 +++++++++++++ .../templates/so/so-snort-template.json.jinja | 13 +++++++++++++ .../templates/so/so-snyk-template.json.jinja | 13 +++++++++++++ .../templates/so/so-sonicwall-template.json.jinja | 13 +++++++++++++ .../templates/so/so-sophos-template.json.jinja | 13 +++++++++++++ .../templates/so/so-squid-template.json.jinja | 13 +++++++++++++ .../templates/so/so-tomcat-template.json.jinja | 13 +++++++++++++ .../templates/so/so-zscaler-template.json.jinja | 13 +++++++++++++ 35 files changed, 455 insertions(+) create mode 100644 salt/elasticsearch/templates/so/so-aws-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-azure-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-barracuda-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-bluecoat-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-cef-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-checkpoint-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-cisco-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-cyberark-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-cylance-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-elasticsearch-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-f5-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-fortinet-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-gcp-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-google_workspace-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-imperva-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-infoblox-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-juniper-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-kibana-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-logstash-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-microsoft-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-misp-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-netflow-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-netscout-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-o365-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-okta-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-proofpoint-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-radware-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-redis-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-snort-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-snyk-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-sonicwall-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-sophos-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-squid-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-tomcat-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-zscaler-template.json.jinja diff --git a/salt/elasticsearch/templates/so/so-aws-template.json.jinja b/salt/elasticsearch/templates/so/so-aws-template.json.jinja new file mode 100644 index 000000000..19b23dfba --- /dev/null +++ b/salt/elasticsearch/templates/so/so-aws-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-aws:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-aws:refresh', '30s') %} +{ + "index_patterns": ["so-aws-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-azure-template.json.jinja b/salt/elasticsearch/templates/so/so-azure-template.json.jinja new file mode 100644 index 000000000..51a266479 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-azure-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-azure:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-azure:refresh', '30s') %} +{ + "index_patterns": ["so-azure-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-barracuda-template.json.jinja b/salt/elasticsearch/templates/so/so-barracuda-template.json.jinja new file mode 100644 index 000000000..66967d6d1 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-barracuda-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-barracuda:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-barracuda:refresh', '30s') %} +{ + "index_patterns": ["so-barracuda-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-bluecoat-template.json.jinja b/salt/elasticsearch/templates/so/so-bluecoat-template.json.jinja new file mode 100644 index 000000000..b1714183e --- /dev/null +++ b/salt/elasticsearch/templates/so/so-bluecoat-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-bluecoat:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-bluecoat:refresh', '30s') %} +{ + "index_patterns": ["so-bluecoat-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-cef-template.json.jinja b/salt/elasticsearch/templates/so/so-cef-template.json.jinja new file mode 100644 index 000000000..0081d42e1 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-cef-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-cef:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-cef:refresh', '30s') %} +{ + "index_patterns": ["so-cef-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-checkpoint-template.json.jinja b/salt/elasticsearch/templates/so/so-checkpoint-template.json.jinja new file mode 100644 index 000000000..5d41946cf --- /dev/null +++ b/salt/elasticsearch/templates/so/so-checkpoint-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-checkpoint:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-checkpoint:refresh', '30s') %} +{ + "index_patterns": ["so-checkpoint-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-cisco-template.json.jinja b/salt/elasticsearch/templates/so/so-cisco-template.json.jinja new file mode 100644 index 000000000..e6e6d14d0 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-cisco-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-cisco:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-cisco:refresh', '30s') %} +{ + "index_patterns": ["so-cisco-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-cyberark-template.json.jinja b/salt/elasticsearch/templates/so/so-cyberark-template.json.jinja new file mode 100644 index 000000000..1647d600f --- /dev/null +++ b/salt/elasticsearch/templates/so/so-cyberark-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-cyberark:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-cyberark:refresh', '30s') %} +{ + "index_patterns": ["so-cyberark-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-cylance-template.json.jinja b/salt/elasticsearch/templates/so/so-cylance-template.json.jinja new file mode 100644 index 000000000..4ba7d0316 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-cylance-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-cylance:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-cylance:refresh', '30s') %} +{ + "index_patterns": ["so-cylance-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-elasticsearch-template.json.jinja b/salt/elasticsearch/templates/so/so-elasticsearch-template.json.jinja new file mode 100644 index 000000000..16aaaec13 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-elasticsearch-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-elasticsearch:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-elasticsearch:refresh', '30s') %} +{ + "index_patterns": ["so-elasticsearch-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-f5-template.json.jinja b/salt/elasticsearch/templates/so/so-f5-template.json.jinja new file mode 100644 index 000000000..682a37c59 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-f5-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-f5:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-f5:refresh', '30s') %} +{ + "index_patterns": ["so-f5-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-fortinet-template.json.jinja b/salt/elasticsearch/templates/so/so-fortinet-template.json.jinja new file mode 100644 index 000000000..616607f52 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-fortinet-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zeek:refresh', '30s') %} +{ + "index_patterns": ["so-zeek-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-gcp-template.json.jinja b/salt/elasticsearch/templates/so/so-gcp-template.json.jinja new file mode 100644 index 000000000..4f1db4f20 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-gcp-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-gcp:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-gcp:refresh', '30s') %} +{ + "index_patterns": ["so-gcp-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-google_workspace-template.json.jinja b/salt/elasticsearch/templates/so/so-google_workspace-template.json.jinja new file mode 100644 index 000000000..5ae26780a --- /dev/null +++ b/salt/elasticsearch/templates/so/so-google_workspace-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-google_workspace:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-google_workspace:refresh', '30s') %} +{ + "index_patterns": ["so-google_workspace-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-imperva-template.json.jinja b/salt/elasticsearch/templates/so/so-imperva-template.json.jinja new file mode 100644 index 000000000..1f574f33a --- /dev/null +++ b/salt/elasticsearch/templates/so/so-imperva-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-imperva:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-imperva:refresh', '30s') %} +{ + "index_patterns": ["so-imperva-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-infoblox-template.json.jinja b/salt/elasticsearch/templates/so/so-infoblox-template.json.jinja new file mode 100644 index 000000000..de613de7f --- /dev/null +++ b/salt/elasticsearch/templates/so/so-infoblox-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-infoblox:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-infoblox:refresh', '30s') %} +{ + "index_patterns": ["so-infoblox-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-juniper-template.json.jinja b/salt/elasticsearch/templates/so/so-juniper-template.json.jinja new file mode 100644 index 000000000..f637271a9 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-juniper-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-juniper:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-juniper:refresh', '30s') %} +{ + "index_patterns": ["so-juniper-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-kibana-template.json.jinja b/salt/elasticsearch/templates/so/so-kibana-template.json.jinja new file mode 100644 index 000000000..fe2004b0e --- /dev/null +++ b/salt/elasticsearch/templates/so/so-kibana-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-kibana:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-kibana:refresh', '30s') %} +{ + "index_patterns": ["so-kibana-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-logstash-template.json.jinja b/salt/elasticsearch/templates/so/so-logstash-template.json.jinja new file mode 100644 index 000000000..2cf0aba42 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-logstash-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-logstash:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-logstash:refresh', '30s') %} +{ + "index_patterns": ["so-logstash-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-microsoft-template.json.jinja b/salt/elasticsearch/templates/so/so-microsoft-template.json.jinja new file mode 100644 index 000000000..3493ccbb2 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-microsoft-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-microsoft:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-microsoft:refresh', '30s') %} +{ + "index_patterns": ["so-microsoft-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-misp-template.json.jinja b/salt/elasticsearch/templates/so/so-misp-template.json.jinja new file mode 100644 index 000000000..67af1efde --- /dev/null +++ b/salt/elasticsearch/templates/so/so-misp-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-misp:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-misp:refresh', '30s') %} +{ + "index_patterns": ["so-misp-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-netflow-template.json.jinja b/salt/elasticsearch/templates/so/so-netflow-template.json.jinja new file mode 100644 index 000000000..62c0972bf --- /dev/null +++ b/salt/elasticsearch/templates/so/so-netflow-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-netflow:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-netflow:refresh', '30s') %} +{ + "index_patterns": ["so-netflow-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-netscout-template.json.jinja b/salt/elasticsearch/templates/so/so-netscout-template.json.jinja new file mode 100644 index 000000000..1dfe336d9 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-netscout-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-netscout:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-netscout:refresh', '30s') %} +{ + "index_patterns": ["so-netscout-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-o365-template.json.jinja b/salt/elasticsearch/templates/so/so-o365-template.json.jinja new file mode 100644 index 000000000..c1f4826f4 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-o365-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-o365:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-o365:refresh', '30s') %} +{ + "index_patterns": ["so-o365-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-okta-template.json.jinja b/salt/elasticsearch/templates/so/so-okta-template.json.jinja new file mode 100644 index 000000000..a4f2df44e --- /dev/null +++ b/salt/elasticsearch/templates/so/so-okta-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-okta:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-okta:refresh', '30s') %} +{ + "index_patterns": ["so-okta-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-proofpoint-template.json.jinja b/salt/elasticsearch/templates/so/so-proofpoint-template.json.jinja new file mode 100644 index 000000000..d7b0ecbe2 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-proofpoint-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-proofpoint:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-proofpoint:refresh', '30s') %} +{ + "index_patterns": ["so-proofpoint-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-radware-template.json.jinja b/salt/elasticsearch/templates/so/so-radware-template.json.jinja new file mode 100644 index 000000000..4efef6a4f --- /dev/null +++ b/salt/elasticsearch/templates/so/so-radware-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-radware:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-radware:refresh', '30s') %} +{ + "index_patterns": ["so-radware-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-redis-template.json.jinja b/salt/elasticsearch/templates/so/so-redis-template.json.jinja new file mode 100644 index 000000000..616607f52 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-redis-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zeek:refresh', '30s') %} +{ + "index_patterns": ["so-zeek-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-snort-template.json.jinja b/salt/elasticsearch/templates/so/so-snort-template.json.jinja new file mode 100644 index 000000000..325f86bde --- /dev/null +++ b/salt/elasticsearch/templates/so/so-snort-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-snort:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-snort:refresh', '30s') %} +{ + "index_patterns": ["so-snort-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-snyk-template.json.jinja b/salt/elasticsearch/templates/so/so-snyk-template.json.jinja new file mode 100644 index 000000000..616607f52 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-snyk-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zeek:refresh', '30s') %} +{ + "index_patterns": ["so-zeek-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-sonicwall-template.json.jinja b/salt/elasticsearch/templates/so/so-sonicwall-template.json.jinja new file mode 100644 index 000000000..b912e4d66 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-sonicwall-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-sonicwall:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-sonicwall:refresh', '30s') %} +{ + "index_patterns": ["so-sonicwall-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-sophos-template.json.jinja b/salt/elasticsearch/templates/so/so-sophos-template.json.jinja new file mode 100644 index 000000000..616607f52 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-sophos-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zeek:refresh', '30s') %} +{ + "index_patterns": ["so-zeek-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-squid-template.json.jinja b/salt/elasticsearch/templates/so/so-squid-template.json.jinja new file mode 100644 index 000000000..616607f52 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-squid-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zeek:refresh', '30s') %} +{ + "index_patterns": ["so-zeek-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-tomcat-template.json.jinja b/salt/elasticsearch/templates/so/so-tomcat-template.json.jinja new file mode 100644 index 000000000..797e71bcf --- /dev/null +++ b/salt/elasticsearch/templates/so/so-tomcat-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-tomcat:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-tomcat:refresh', '30s') %} +{ + "index_patterns": ["so-tomcat-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-zscaler-template.json.jinja b/salt/elasticsearch/templates/so/so-zscaler-template.json.jinja new file mode 100644 index 000000000..01d1cab2d --- /dev/null +++ b/salt/elasticsearch/templates/so/so-zscaler-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zscaler:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zscaler:refresh', '30s') %} +{ + "index_patterns": ["so-zscaler-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} From a27263435abd126f2f053dfc6bc237627e73d389 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 27 Aug 2021 14:41:04 -0400 Subject: [PATCH 07/61] Add Templates for all filebeat modules --- salt/elasticsearch/templates/so/so-snyk-template.json.jinja | 6 +++--- .../templates/so/so-sophos-template.json.jinja | 6 +++--- .../elasticsearch/templates/so/so-squid-template.json.jinja | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/salt/elasticsearch/templates/so/so-snyk-template.json.jinja b/salt/elasticsearch/templates/so/so-snyk-template.json.jinja index 616607f52..0c2d291cc 100644 --- a/salt/elasticsearch/templates/so/so-snyk-template.json.jinja +++ b/salt/elasticsearch/templates/so/so-snyk-template.json.jinja @@ -1,8 +1,8 @@ -{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:shards', 1) %} +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-snyk:shards', 1) %} {%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} -{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zeek:refresh', '30s') %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-snyk:refresh', '30s') %} { - "index_patterns": ["so-zeek-*"], + "index_patterns": ["so-snyk-*"], "version":50001, "order":11, "settings":{ diff --git a/salt/elasticsearch/templates/so/so-sophos-template.json.jinja b/salt/elasticsearch/templates/so/so-sophos-template.json.jinja index 616607f52..689e19999 100644 --- a/salt/elasticsearch/templates/so/so-sophos-template.json.jinja +++ b/salt/elasticsearch/templates/so/so-sophos-template.json.jinja @@ -1,8 +1,8 @@ -{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:shards', 1) %} +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-sophos:shards', 1) %} {%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} -{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zeek:refresh', '30s') %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-sophos:refresh', '30s') %} { - "index_patterns": ["so-zeek-*"], + "index_patterns": ["so-sophos-*"], "version":50001, "order":11, "settings":{ diff --git a/salt/elasticsearch/templates/so/so-squid-template.json.jinja b/salt/elasticsearch/templates/so/so-squid-template.json.jinja index 616607f52..9398b8a99 100644 --- a/salt/elasticsearch/templates/so/so-squid-template.json.jinja +++ b/salt/elasticsearch/templates/so/so-squid-template.json.jinja @@ -1,8 +1,8 @@ -{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:shards', 1) %} +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-squid:shards', 1) %} {%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} -{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zeek:refresh', '30s') %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-squid:refresh', '30s') %} { - "index_patterns": ["so-zeek-*"], + "index_patterns": ["so-squid-*"], "version":50001, "order":11, "settings":{ From 72859adb13b049d8e6317ae386f3fef4fb194e11 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 27 Aug 2021 15:23:01 -0400 Subject: [PATCH 08/61] Fix typo in so-checkin --- salt/common/tools/sbin/so-checkin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-checkin b/salt/common/tools/sbin/so-checkin index c70701b71..4f0583906 100755 --- a/salt/common/tools/sbin/so-checkin +++ b/salt/common/tools/sbin/so-checkin @@ -17,4 +17,4 @@ . /usr/sbin/so-common -salt-call state.highstate -linfo +salt-call state.highstate -l info From 2b34da0fee25440eaf29fd41e2c7a47af9c513ca Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 30 Aug 2021 12:32:44 -0400 Subject: [PATCH 09/61] Update HOTFIX --- HOTFIX | 1 - 1 file changed, 1 deletion(-) diff --git a/HOTFIX b/HOTFIX index 100732042..c9e34953f 100644 --- a/HOTFIX +++ b/HOTFIX @@ -1,2 +1 @@ - CURATOR GRAFANA_DASH_ALLOW From bfd632e20a6a8b5ea26caff27db08d9a61f1b49a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 30 Aug 2021 14:21:13 -0400 Subject: [PATCH 10/61] Add wazuh to exclude arg when running yum update --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 7bbaa1fda..9a64a561e 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2795,7 +2795,7 @@ update_sudoers() { update_packages() { if [ "$OS" = 'centos' ]; then yum repolist >> /dev/null - yum -y update --exclude=salt* >> "$setup_log" + yum -y update --exclude=salt*,wazuh* >> "$setup_log" else retry 50 10 "apt-get -y update" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-get -y upgrade" >> "$setup_log" 2>&1 || exit 1 From 92858cd13ab7d2e404ea45b37f67566ddd3259c2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 30 Aug 2021 17:38:29 -0400 Subject: [PATCH 11/61] Update HOTFIX --- HOTFIX | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HOTFIX b/HOTFIX index c9e34953f..e87aa6dbd 100644 --- a/HOTFIX +++ b/HOTFIX @@ -1 +1 @@ -CURATOR GRAFANA_DASH_ALLOW +CURATOR GRAFANA_DASH_ALLOW WAZUH From 4fe0a1d7b436d7393bfdac13e9251cb7002f3357 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 08:39:37 -0400 Subject: [PATCH 12/61] 2.3.70 WAZUH Hotfix sigs --- VERIFY_ISO.md | 22 +++++++++++----------- sigs/securityonion-2.3.70-WAZUH.iso.sig | Bin 0 -> 543 bytes 2 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 sigs/securityonion-2.3.70-WAZUH.iso.sig diff --git a/VERIFY_ISO.md b/VERIFY_ISO.md index 2be975430..660f1ef08 100644 --- a/VERIFY_ISO.md +++ b/VERIFY_ISO.md @@ -1,18 +1,18 @@ -### 2.3.70-GRAFANA ISO image built on 2021/08/23 +### 2.3.70-WAZUH ISO image built on 2021/08/23 ### Download and Verify -2.3.70-GRAFANA ISO image: -https://download.securityonion.net/file/securityonion/securityonion-2.3.70-GRAFANA.iso +2.3.70-WAZUH ISO image: +https://download.securityonion.net/file/securityonion/securityonion-2.3.70-WAZUH.iso -MD5: A16683FC8F2151C290E359FC6066B1F2 -SHA1: A93329C103CCCE665968F246163FBE5D41EF0510 -SHA256: 3ED0177CADF203324363916AA240A10C58DC3E9044A9ADE173A80674701A50A3 +MD5: CEDEF3C38089896C252F9E3C75F7CB15 +SHA1: FB420115C72DABDEB87C8B27F26E862C94628057 +SHA256: CC3E75A97163E9CD255DA0D9C3EB11922FA045651827F291025398943C1BC230 Signature for ISO image: -https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.70-GRAFANA.iso.sig +https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.70-WAZUH.iso.sig Signing key: https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS @@ -26,22 +26,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/ma Download the signature file for the ISO: ``` -wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.70-GRAFANA.iso.sig +wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.70-WAZUH.iso.sig ``` Download the ISO image: ``` -wget https://download.securityonion.net/file/securityonion/securityonion-2.3.70-GRAFANA.iso +wget https://download.securityonion.net/file/securityonion/securityonion-2.3.70-WAZUH.iso ``` Verify the downloaded ISO image using the signature file: ``` -gpg --verify securityonion-2.3.70-GRAFANA.iso.sig securityonion-2.3.70-GRAFANA.iso +gpg --verify securityonion-2.3.70-WAZUH.iso.sig securityonion-2.3.70-WAZUH.iso ``` The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Mon 23 Aug 2021 01:43:00 PM EDT using RSA key ID FE507013 +gpg: Signature made Mon 30 Aug 2021 06:13:14 PM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/sigs/securityonion-2.3.70-WAZUH.iso.sig b/sigs/securityonion-2.3.70-WAZUH.iso.sig new file mode 100644 index 0000000000000000000000000000000000000000..43ce74d15f27fd7404032466521b8f41493d216f GIT binary patch literal 543 zcmV+)0^t3L0vrSY0RjL91p;9$SNZ@72@re`V7LBIa1&ji5CFP43k_8LGaz5WZEq#U zjfxe1>pW9wHJSpK$w&`hIs|gd_hGkOa0Qcjg9u49g;ggxVp>DrKLfl{0N)Lu)vqNUO&}dp0)O@i(&g%Yp!t-=1v7XcqA8m- zNuGLYl+9n8BB~E5{WOI<&Yh%{#!Ri4Is5Sa+c2_#?^po`n{RAG zE>jx($L2Aywe2R-8tb>tYi`!r+iI*8O;Y#iy2!*FFHMy}aL_jo$chB>b^nUi7#~Of zXuk6R{56wMf=YM-PurQnsc0}3gEvnRvtd!qz?St+wh;T9d4Zc%9JTO{#9`lNm*1?WLA7L9 hGjwSAR+NL){>p*2%?PW}Wj1OMZdTyolF!u`t>a4(0qXz& literal 0 HcmV?d00001 From 77b82bf2c09aa6bd8bba765b552f3ce6a03a7cff Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 10:01:32 -0400 Subject: [PATCH 13/61] Update VERIFY_ISO.md --- VERIFY_ISO.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERIFY_ISO.md b/VERIFY_ISO.md index 660f1ef08..f6cc31508 100644 --- a/VERIFY_ISO.md +++ b/VERIFY_ISO.md @@ -1,4 +1,4 @@ -### 2.3.70-WAZUH ISO image built on 2021/08/23 +### 2.3.70-WAZUH ISO image built on 2021/08/30 From 9605eda55959ca74e09b8c695211251caec016c7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 10:49:39 -0400 Subject: [PATCH 14/61] Close Curators for all modules --- salt/curator/files/action/so-aws-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-azure-close.yml | 29 +++++++++++++++++++ .../files/action/so-barracuda-close.yml | 29 +++++++++++++++++++ .../files/action/so-bluecoat-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-cef-close.yml | 29 +++++++++++++++++++ .../files/action/so-checkpoint-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-cisco-close.yml | 29 +++++++++++++++++++ .../files/action/so-cyberark-close.yml | 29 +++++++++++++++++++ .../curator/files/action/so-cylance-close.yml | 29 +++++++++++++++++++ .../files/action/so-elasticsearch-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-f5-close.yml | 29 +++++++++++++++++++ .../files/action/so-fortinet-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-gcp-close.yml | 29 +++++++++++++++++++ .../action/so-google_workspace-close.yml | 29 +++++++++++++++++++ .../curator/files/action/so-imperva-close.yml | 29 +++++++++++++++++++ .../files/action/so-infoblox-close.yml | 29 +++++++++++++++++++ .../curator/files/action/so-juniper-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-kibana-close.yml | 29 +++++++++++++++++++ .../files/action/so-logstash-close.yml | 29 +++++++++++++++++++ .../files/action/so-microsoft-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-misp-close.yml | 29 +++++++++++++++++++ .../curator/files/action/so-netflow-close.yml | 29 +++++++++++++++++++ .../files/action/so-netscout-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-o365-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-okta-close.yml | 29 +++++++++++++++++++ .../files/action/so-proofpoint-close.yml | 29 +++++++++++++++++++ .../curator/files/action/so-radware-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-redis-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-snort-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-snyk-close.yml | 29 +++++++++++++++++++ .../files/action/so-sonicwall-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-sophos-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-tomcat-close.yml | 29 +++++++++++++++++++ .../curator/files/action/so-zscaler-close.yml | 29 +++++++++++++++++++ 34 files changed, 986 insertions(+) create mode 100644 salt/curator/files/action/so-aws-close.yml create mode 100644 salt/curator/files/action/so-azure-close.yml create mode 100644 salt/curator/files/action/so-barracuda-close.yml create mode 100644 salt/curator/files/action/so-bluecoat-close.yml create mode 100644 salt/curator/files/action/so-cef-close.yml create mode 100644 salt/curator/files/action/so-checkpoint-close.yml create mode 100644 salt/curator/files/action/so-cisco-close.yml create mode 100644 salt/curator/files/action/so-cyberark-close.yml create mode 100644 salt/curator/files/action/so-cylance-close.yml create mode 100644 salt/curator/files/action/so-elasticsearch-close.yml create mode 100644 salt/curator/files/action/so-f5-close.yml create mode 100644 salt/curator/files/action/so-fortinet-close.yml create mode 100644 salt/curator/files/action/so-gcp-close.yml create mode 100644 salt/curator/files/action/so-google_workspace-close.yml create mode 100644 salt/curator/files/action/so-imperva-close.yml create mode 100644 salt/curator/files/action/so-infoblox-close.yml create mode 100644 salt/curator/files/action/so-juniper-close.yml create mode 100644 salt/curator/files/action/so-kibana-close.yml create mode 100644 salt/curator/files/action/so-logstash-close.yml create mode 100644 salt/curator/files/action/so-microsoft-close.yml create mode 100644 salt/curator/files/action/so-misp-close.yml create mode 100644 salt/curator/files/action/so-netflow-close.yml create mode 100644 salt/curator/files/action/so-netscout-close.yml create mode 100644 salt/curator/files/action/so-o365-close.yml create mode 100644 salt/curator/files/action/so-okta-close.yml create mode 100644 salt/curator/files/action/so-proofpoint-close.yml create mode 100644 salt/curator/files/action/so-radware-close.yml create mode 100644 salt/curator/files/action/so-redis-close.yml create mode 100644 salt/curator/files/action/so-snort-close.yml create mode 100644 salt/curator/files/action/so-snyk-close.yml create mode 100644 salt/curator/files/action/so-sonicwall-close.yml create mode 100644 salt/curator/files/action/so-sophos-close.yml create mode 100644 salt/curator/files/action/so-tomcat-close.yml create mode 100644 salt/curator/files/action/so-zscaler-close.yml diff --git a/salt/curator/files/action/so-aws-close.yml b/salt/curator/files/action/so-aws-close.yml new file mode 100644 index 000000000..44f0bfa97 --- /dev/null +++ b/salt/curator/files/action/so-aws-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-aws:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close aws indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-aws.*|so-aws.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-azure-close.yml b/salt/curator/files/action/so-azure-close.yml new file mode 100644 index 000000000..901b2c0ba --- /dev/null +++ b/salt/curator/files/action/so-azure-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-azure:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close azure indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-azure.*|so-azure.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-barracuda-close.yml b/salt/curator/files/action/so-barracuda-close.yml new file mode 100644 index 000000000..496832db7 --- /dev/null +++ b/salt/curator/files/action/so-barracuda-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-barracuda:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close barracuda indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-barracuda.*|so-barracuda.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-bluecoat-close.yml b/salt/curator/files/action/so-bluecoat-close.yml new file mode 100644 index 000000000..86d9277eb --- /dev/null +++ b/salt/curator/files/action/so-bluecoat-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-bluecoat:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close bluecoat indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-bluecoat.*|so-bluecoat.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-cef-close.yml b/salt/curator/files/action/so-cef-close.yml new file mode 100644 index 000000000..49e07f764 --- /dev/null +++ b/salt/curator/files/action/so-cef-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-cef:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close cef indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-cef.*|so-cef.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-checkpoint-close.yml b/salt/curator/files/action/so-checkpoint-close.yml new file mode 100644 index 000000000..cffdf6473 --- /dev/null +++ b/salt/curator/files/action/so-checkpoint-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-checkpoint:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close checkpoint indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-checkpoint.*|so-checkpoint.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-cisco-close.yml b/salt/curator/files/action/so-cisco-close.yml new file mode 100644 index 000000000..cd1faade1 --- /dev/null +++ b/salt/curator/files/action/so-cisco-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-cisco:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close cisco indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-cisco.*|so-cisco.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-cyberark-close.yml b/salt/curator/files/action/so-cyberark-close.yml new file mode 100644 index 000000000..e352e8355 --- /dev/null +++ b/salt/curator/files/action/so-cyberark-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-cyberark:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close cyberark indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-cyberark.*|so-cyberark.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-cylance-close.yml b/salt/curator/files/action/so-cylance-close.yml new file mode 100644 index 000000000..d808569fb --- /dev/null +++ b/salt/curator/files/action/so-cylance-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-cylance:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close cylance indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-cylance.*|so-cylance.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-elasticsearch-close.yml b/salt/curator/files/action/so-elasticsearch-close.yml new file mode 100644 index 000000000..3c4ff0dac --- /dev/null +++ b/salt/curator/files/action/so-elasticsearch-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-elasticsearch:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close elasticsearch indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-elasticsearch.*|so-elasticsearch.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-f5-close.yml b/salt/curator/files/action/so-f5-close.yml new file mode 100644 index 000000000..e1cdb48a1 --- /dev/null +++ b/salt/curator/files/action/so-f5-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-f5:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close f5 indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-f5.*|so-f5.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-fortinet-close.yml b/salt/curator/files/action/so-fortinet-close.yml new file mode 100644 index 000000000..e11fb86c6 --- /dev/null +++ b/salt/curator/files/action/so-fortinet-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-fortinet:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close fortinet indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-fortinet.*|so-fortinet.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-gcp-close.yml b/salt/curator/files/action/so-gcp-close.yml new file mode 100644 index 000000000..f9dd0af24 --- /dev/null +++ b/salt/curator/files/action/so-gcp-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-gcp:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close gcp indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-gcp.*|so-gcp.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-google_workspace-close.yml b/salt/curator/files/action/so-google_workspace-close.yml new file mode 100644 index 000000000..1ecda5893 --- /dev/null +++ b/salt/curator/files/action/so-google_workspace-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-google_workspace:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close google_workspace indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-google_workspace.*|so-google_workspace.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-imperva-close.yml b/salt/curator/files/action/so-imperva-close.yml new file mode 100644 index 000000000..55ec2e472 --- /dev/null +++ b/salt/curator/files/action/so-imperva-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-imperva:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close imperva indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-imperva.*|so-imperva.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-infoblox-close.yml b/salt/curator/files/action/so-infoblox-close.yml new file mode 100644 index 000000000..9fd4c5070 --- /dev/null +++ b/salt/curator/files/action/so-infoblox-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-infoblox:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close infoblox indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-infoblox.*|so-infoblox.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-juniper-close.yml b/salt/curator/files/action/so-juniper-close.yml new file mode 100644 index 000000000..466a51eca --- /dev/null +++ b/salt/curator/files/action/so-juniper-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-juniper:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close juniper indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-juniper.*|so-juniper.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-kibana-close.yml b/salt/curator/files/action/so-kibana-close.yml new file mode 100644 index 000000000..05583d853 --- /dev/null +++ b/salt/curator/files/action/so-kibana-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close IDS indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-ids.*|so-ids.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-logstash-close.yml b/salt/curator/files/action/so-logstash-close.yml new file mode 100644 index 000000000..23787e237 --- /dev/null +++ b/salt/curator/files/action/so-logstash-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-logstash:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close logstash indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-logstash.*|so-logstash.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-microsoft-close.yml b/salt/curator/files/action/so-microsoft-close.yml new file mode 100644 index 000000000..f4eaf738f --- /dev/null +++ b/salt/curator/files/action/so-microsoft-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-microsoft:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close microsoft indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-microsoft.*|so-microsoft.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-misp-close.yml b/salt/curator/files/action/so-misp-close.yml new file mode 100644 index 000000000..05583d853 --- /dev/null +++ b/salt/curator/files/action/so-misp-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close IDS indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-ids.*|so-ids.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-netflow-close.yml b/salt/curator/files/action/so-netflow-close.yml new file mode 100644 index 000000000..cc9ade28d --- /dev/null +++ b/salt/curator/files/action/so-netflow-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-netflow:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close netflow indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-netflow.*|so-netflow.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-netscout-close.yml b/salt/curator/files/action/so-netscout-close.yml new file mode 100644 index 000000000..05583d853 --- /dev/null +++ b/salt/curator/files/action/so-netscout-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close IDS indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-ids.*|so-ids.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-o365-close.yml b/salt/curator/files/action/so-o365-close.yml new file mode 100644 index 000000000..4dece060f --- /dev/null +++ b/salt/curator/files/action/so-o365-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-o365:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close o365 indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-o365.*|so-o365.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-okta-close.yml b/salt/curator/files/action/so-okta-close.yml new file mode 100644 index 000000000..10f7e4b60 --- /dev/null +++ b/salt/curator/files/action/so-okta-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-okta:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close okta indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-okta.*|so-okta.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-proofpoint-close.yml b/salt/curator/files/action/so-proofpoint-close.yml new file mode 100644 index 000000000..888c9fc64 --- /dev/null +++ b/salt/curator/files/action/so-proofpoint-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-proofpoint:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close proofpoint indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-proofpoint.*|so-proofpoint.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-radware-close.yml b/salt/curator/files/action/so-radware-close.yml new file mode 100644 index 000000000..05583d853 --- /dev/null +++ b/salt/curator/files/action/so-radware-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close IDS indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-ids.*|so-ids.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-redis-close.yml b/salt/curator/files/action/so-redis-close.yml new file mode 100644 index 000000000..b69935f21 --- /dev/null +++ b/salt/curator/files/action/so-redis-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-redis:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close redis indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-redis.*|so-redis.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-snort-close.yml b/salt/curator/files/action/so-snort-close.yml new file mode 100644 index 000000000..8f6209255 --- /dev/null +++ b/salt/curator/files/action/so-snort-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-snort:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close snort indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-snort.*|so-snort.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-snyk-close.yml b/salt/curator/files/action/so-snyk-close.yml new file mode 100644 index 000000000..05583d853 --- /dev/null +++ b/salt/curator/files/action/so-snyk-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close IDS indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-ids.*|so-ids.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-sonicwall-close.yml b/salt/curator/files/action/so-sonicwall-close.yml new file mode 100644 index 000000000..9cc23d3af --- /dev/null +++ b/salt/curator/files/action/so-sonicwall-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-sonicwall:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close sonicwall indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-sonicwall.*|so-sonicwall.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-sophos-close.yml b/salt/curator/files/action/so-sophos-close.yml new file mode 100644 index 000000000..b7574b996 --- /dev/null +++ b/salt/curator/files/action/so-sophos-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-sophos:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close sophos indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-sophos.*|so-sophos.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-tomcat-close.yml b/salt/curator/files/action/so-tomcat-close.yml new file mode 100644 index 000000000..ea0d95b0a --- /dev/null +++ b/salt/curator/files/action/so-tomcat-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-tomcat:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close tomcat indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-tomcat.*|so-tomcat.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-zscaler-close.yml b/salt/curator/files/action/so-zscaler-close.yml new file mode 100644 index 000000000..5a008a27d --- /dev/null +++ b/salt/curator/files/action/so-zscaler-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-zscaler:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close zscaler indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-zscaler.*|so-zscaler.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: From 3425a0fe78571b493495017d1df32a93e56edc19 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 11:12:21 -0400 Subject: [PATCH 15/61] Delete Curators for all modules --- salt/curator/files/action/so-aws-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-azure-delete.yml | 29 +++++++++++++++++++ .../files/action/so-barracuda-delete.yml | 29 +++++++++++++++++++ .../files/action/so-bluecoat-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-cef-delete.yml | 29 +++++++++++++++++++ .../files/action/so-checkpoint-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-cisco-delete.yml | 29 +++++++++++++++++++ .../files/action/so-cyberark-delete.yml | 29 +++++++++++++++++++ .../files/action/so-cylance-delete.yml | 29 +++++++++++++++++++ .../files/action/so-elasticsearch-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-f5-delete.yml | 29 +++++++++++++++++++ .../files/action/so-fortinet-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-gcp-delete.yml | 29 +++++++++++++++++++ .../files/action/so-google_workspace.yml | 29 +++++++++++++++++++ .../files/action/so-imperva-delete.yml | 29 +++++++++++++++++++ .../files/action/so-infoblox-delete.yml | 29 +++++++++++++++++++ .../files/action/so-juniper-delete.yml | 29 +++++++++++++++++++ .../curator/files/action/so-kibana-delete.yml | 29 +++++++++++++++++++ .../files/action/so-logstash-delete.yml | 29 +++++++++++++++++++ .../files/action/so-microsoft-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-misp-close.yml | 6 ++-- salt/curator/files/action/so-misp-delete.yml | 29 +++++++++++++++++++ .../files/action/so-netflow-delete.yml | 29 +++++++++++++++++++ .../files/action/so-netscout-close.yml | 6 ++-- salt/curator/files/action/so-o365-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-okta.delete.yml | 29 +++++++++++++++++++ .../files/action/so-proofpoint-delete.yml | 29 +++++++++++++++++++ .../curator/files/action/so-radware-close.yml | 6 ++-- .../files/action/so-radware-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-redis-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-snort-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-snyk-close.yml | 6 ++-- salt/curator/files/action/so-snyk-delete.yml | 29 +++++++++++++++++++ .../files/action/so-sonicwall-delete.yml | 29 +++++++++++++++++++ .../curator/files/action/so-sophos-delete.yml | 29 +++++++++++++++++++ .../curator/files/action/so-tomcat-delete.yml | 29 +++++++++++++++++++ .../files/action/so-zscaler-delete.yml | 29 +++++++++++++++++++ 37 files changed, 969 insertions(+), 12 deletions(-) create mode 100644 salt/curator/files/action/so-aws-delete.yml create mode 100644 salt/curator/files/action/so-azure-delete.yml create mode 100644 salt/curator/files/action/so-barracuda-delete.yml create mode 100644 salt/curator/files/action/so-bluecoat-delete.yml create mode 100644 salt/curator/files/action/so-cef-delete.yml create mode 100644 salt/curator/files/action/so-checkpoint-delete.yml create mode 100644 salt/curator/files/action/so-cisco-delete.yml create mode 100644 salt/curator/files/action/so-cyberark-delete.yml create mode 100644 salt/curator/files/action/so-cylance-delete.yml create mode 100644 salt/curator/files/action/so-elasticsearch-delete.yml create mode 100644 salt/curator/files/action/so-f5-delete.yml create mode 100644 salt/curator/files/action/so-fortinet-delete.yml create mode 100644 salt/curator/files/action/so-gcp-delete.yml create mode 100644 salt/curator/files/action/so-google_workspace.yml create mode 100644 salt/curator/files/action/so-imperva-delete.yml create mode 100644 salt/curator/files/action/so-infoblox-delete.yml create mode 100644 salt/curator/files/action/so-juniper-delete.yml create mode 100644 salt/curator/files/action/so-kibana-delete.yml create mode 100644 salt/curator/files/action/so-logstash-delete.yml create mode 100644 salt/curator/files/action/so-microsoft-delete.yml create mode 100644 salt/curator/files/action/so-misp-delete.yml create mode 100644 salt/curator/files/action/so-netflow-delete.yml create mode 100644 salt/curator/files/action/so-o365-delete.yml create mode 100644 salt/curator/files/action/so-okta.delete.yml create mode 100644 salt/curator/files/action/so-proofpoint-delete.yml create mode 100644 salt/curator/files/action/so-radware-delete.yml create mode 100644 salt/curator/files/action/so-redis-delete.yml create mode 100644 salt/curator/files/action/so-snort-delete.yml create mode 100644 salt/curator/files/action/so-snyk-delete.yml create mode 100644 salt/curator/files/action/so-sonicwall-delete.yml create mode 100644 salt/curator/files/action/so-sophos-delete.yml create mode 100644 salt/curator/files/action/so-tomcat-delete.yml create mode 100644 salt/curator/files/action/so-zscaler-delete.yml diff --git a/salt/curator/files/action/so-aws-delete.yml b/salt/curator/files/action/so-aws-delete.yml new file mode 100644 index 000000000..a67ee88b8 --- /dev/null +++ b/salt/curator/files/action/so-aws-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete aws indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-aws.*|so-aws.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-azure-delete.yml b/salt/curator/files/action/so-azure-delete.yml new file mode 100644 index 000000000..102a69d3d --- /dev/null +++ b/salt/curator/files/action/so-azure-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-azure:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete azure indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-azure.*|so-azure.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-barracuda-delete.yml b/salt/curator/files/action/so-barracuda-delete.yml new file mode 100644 index 000000000..49d472618 --- /dev/null +++ b/salt/curator/files/action/so-barracuda-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-barracuda:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete barracuda indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-barracuda.*|so-barracuda.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-bluecoat-delete.yml b/salt/curator/files/action/so-bluecoat-delete.yml new file mode 100644 index 000000000..318624416 --- /dev/null +++ b/salt/curator/files/action/so-bluecoat-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-bluecoat:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete bluecoat indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-bluecoat.*|so-bluecoat.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-cef-delete.yml b/salt/curator/files/action/so-cef-delete.yml new file mode 100644 index 000000000..0ee7d6501 --- /dev/null +++ b/salt/curator/files/action/so-cef-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-cef:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete cef indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-cef.*|so-cef.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-checkpoint-delete.yml b/salt/curator/files/action/so-checkpoint-delete.yml new file mode 100644 index 000000000..d1ac13efe --- /dev/null +++ b/salt/curator/files/action/so-checkpoint-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-checkpoint:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete checkpoint indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-checkpoint.*|so-checkpoint.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-cisco-delete.yml b/salt/curator/files/action/so-cisco-delete.yml new file mode 100644 index 000000000..bb5e06f7f --- /dev/null +++ b/salt/curator/files/action/so-cisco-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-cisco:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete cisco indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-cisco.*|so-cisco.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-cyberark-delete.yml b/salt/curator/files/action/so-cyberark-delete.yml new file mode 100644 index 000000000..784f6881e --- /dev/null +++ b/salt/curator/files/action/so-cyberark-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-cyberark:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete cyberark indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-cyberark.*|so-cyberark.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-cylance-delete.yml b/salt/curator/files/action/so-cylance-delete.yml new file mode 100644 index 000000000..54cf3938b --- /dev/null +++ b/salt/curator/files/action/so-cylance-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-cylance:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete cylance indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-cylance.*|so-cylance.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-elasticsearch-delete.yml b/salt/curator/files/action/so-elasticsearch-delete.yml new file mode 100644 index 000000000..05cc68abe --- /dev/null +++ b/salt/curator/files/action/so-elasticsearch-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-elasticsearch:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete elasticsearch indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-elasticsearch.*|so-elasticsearch.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-f5-delete.yml b/salt/curator/files/action/so-f5-delete.yml new file mode 100644 index 000000000..06704010a --- /dev/null +++ b/salt/curator/files/action/so-f5-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-f5:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete f5 indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-f5.*|so-f5.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-fortinet-delete.yml b/salt/curator/files/action/so-fortinet-delete.yml new file mode 100644 index 000000000..9379e47c2 --- /dev/null +++ b/salt/curator/files/action/so-fortinet-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-fortinet:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete fortinet indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-fortinet.*|so-fortinet.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-gcp-delete.yml b/salt/curator/files/action/so-gcp-delete.yml new file mode 100644 index 000000000..5c8ab33d8 --- /dev/null +++ b/salt/curator/files/action/so-gcp-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-gcp:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete gcp indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-gcp.*|so-gcp.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-google_workspace.yml b/salt/curator/files/action/so-google_workspace.yml new file mode 100644 index 000000000..923feda8f --- /dev/null +++ b/salt/curator/files/action/so-google_workspace.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-google_workspace:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete google_workspace indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-google_workspace.*|so-google_workspace.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-imperva-delete.yml b/salt/curator/files/action/so-imperva-delete.yml new file mode 100644 index 000000000..b5526e2fb --- /dev/null +++ b/salt/curator/files/action/so-imperva-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-imperva:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete imperva indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-imperva.*|so-imperva.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-infoblox-delete.yml b/salt/curator/files/action/so-infoblox-delete.yml new file mode 100644 index 000000000..0a7fdafbe --- /dev/null +++ b/salt/curator/files/action/so-infoblox-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-infoblox:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete infoblox indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-infoblox.*|so-infoblox.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-juniper-delete.yml b/salt/curator/files/action/so-juniper-delete.yml new file mode 100644 index 000000000..18abc86ac --- /dev/null +++ b/salt/curator/files/action/so-juniper-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-juniper:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete juniper indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-juniper.*|so-juniper.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-kibana-delete.yml b/salt/curator/files/action/so-kibana-delete.yml new file mode 100644 index 000000000..5a775b8de --- /dev/null +++ b/salt/curator/files/action/so-kibana-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-kibana:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete kibana indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-kibana.*|so-kibana.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-logstash-delete.yml b/salt/curator/files/action/so-logstash-delete.yml new file mode 100644 index 000000000..d9ff848da --- /dev/null +++ b/salt/curator/files/action/so-logstash-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-logstash:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete logstash indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-logstash.*|so-logstash.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-microsoft-delete.yml b/salt/curator/files/action/so-microsoft-delete.yml new file mode 100644 index 000000000..f1a854c83 --- /dev/null +++ b/salt/curator/files/action/so-microsoft-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-microsoft:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete microsoft indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-microsoft.*|so-microsoft.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-misp-close.yml b/salt/curator/files/action/so-misp-close.yml index 05583d853..e39781353 100644 --- a/salt/curator/files/action/so-misp-close.yml +++ b/salt/curator/files/action/so-misp-close.yml @@ -1,4 +1,4 @@ -{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-misp:close', 30) -%} --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" @@ -10,7 +10,7 @@ actions: 1: action: close description: >- - Close IDS indices older than {{cur_close_days}} days. + Close misp indices older than {{cur_close_days}} days. options: delete_aliases: False timeout_override: @@ -19,7 +19,7 @@ actions: filters: - filtertype: pattern kind: regex - value: '^(logstash-ids.*|so-ids.*)$' + value: '^(logstash-misp.*|so-misp.*)$' - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-misp-delete.yml b/salt/curator/files/action/so-misp-delete.yml new file mode 100644 index 000000000..ceaa9c73d --- /dev/null +++ b/salt/curator/files/action/so-misp-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-misp:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete misp indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-misp.*|so-misp.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-netflow-delete.yml b/salt/curator/files/action/so-netflow-delete.yml new file mode 100644 index 000000000..5bc76ad15 --- /dev/null +++ b/salt/curator/files/action/so-netflow-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-netflow:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete netflow indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-netflow.*|so-netflow.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-netscout-close.yml b/salt/curator/files/action/so-netscout-close.yml index 05583d853..d99374d2f 100644 --- a/salt/curator/files/action/so-netscout-close.yml +++ b/salt/curator/files/action/so-netscout-close.yml @@ -1,4 +1,4 @@ -{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-netscout:close', 30) -%} --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" @@ -10,7 +10,7 @@ actions: 1: action: close description: >- - Close IDS indices older than {{cur_close_days}} days. + Close netscout indices older than {{cur_close_days}} days. options: delete_aliases: False timeout_override: @@ -19,7 +19,7 @@ actions: filters: - filtertype: pattern kind: regex - value: '^(logstash-ids.*|so-ids.*)$' + value: '^(logstash-netscout.*|so-netscout.*)$' - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-o365-delete.yml b/salt/curator/files/action/so-o365-delete.yml new file mode 100644 index 000000000..13c7c1344 --- /dev/null +++ b/salt/curator/files/action/so-o365-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-o365:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete o365 indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-o365.*|so-o365.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-okta.delete.yml b/salt/curator/files/action/so-okta.delete.yml new file mode 100644 index 000000000..1beeb0fc0 --- /dev/null +++ b/salt/curator/files/action/so-okta.delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-okta:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete okta indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-okta.*|so-okta.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-proofpoint-delete.yml b/salt/curator/files/action/so-proofpoint-delete.yml new file mode 100644 index 000000000..903dde204 --- /dev/null +++ b/salt/curator/files/action/so-proofpoint-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-proofpoint:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete proofpoint indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-proofpoint.*|so-proofpoint.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-radware-close.yml b/salt/curator/files/action/so-radware-close.yml index 05583d853..59a7bbafd 100644 --- a/salt/curator/files/action/so-radware-close.yml +++ b/salt/curator/files/action/so-radware-close.yml @@ -1,4 +1,4 @@ -{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-radware:close', 30) -%} --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" @@ -10,7 +10,7 @@ actions: 1: action: close description: >- - Close IDS indices older than {{cur_close_days}} days. + Close radware indices older than {{cur_close_days}} days. options: delete_aliases: False timeout_override: @@ -19,7 +19,7 @@ actions: filters: - filtertype: pattern kind: regex - value: '^(logstash-ids.*|so-ids.*)$' + value: '^(logstash-radware.*|so-radware.*)$' - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-radware-delete.yml b/salt/curator/files/action/so-radware-delete.yml new file mode 100644 index 000000000..1fe09cded --- /dev/null +++ b/salt/curator/files/action/so-radware-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-radware:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete radware indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-radware.*|so-radware.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-redis-delete.yml b/salt/curator/files/action/so-redis-delete.yml new file mode 100644 index 000000000..f6e73dce8 --- /dev/null +++ b/salt/curator/files/action/so-redis-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-redis:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete redis indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-redis.*|so-redis.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-snort-delete.yml b/salt/curator/files/action/so-snort-delete.yml new file mode 100644 index 000000000..50f68988b --- /dev/null +++ b/salt/curator/files/action/so-snort-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-snort:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete snort indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-snort.*|so-snort.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-snyk-close.yml b/salt/curator/files/action/so-snyk-close.yml index 05583d853..e13d8f98d 100644 --- a/salt/curator/files/action/so-snyk-close.yml +++ b/salt/curator/files/action/so-snyk-close.yml @@ -1,4 +1,4 @@ -{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-snyk:close', 30) -%} --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" @@ -10,7 +10,7 @@ actions: 1: action: close description: >- - Close IDS indices older than {{cur_close_days}} days. + Close snyk indices older than {{cur_close_days}} days. options: delete_aliases: False timeout_override: @@ -19,7 +19,7 @@ actions: filters: - filtertype: pattern kind: regex - value: '^(logstash-ids.*|so-ids.*)$' + value: '^(logstash-snyk.*|so-snyk.*)$' - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-snyk-delete.yml b/salt/curator/files/action/so-snyk-delete.yml new file mode 100644 index 000000000..cec0b942f --- /dev/null +++ b/salt/curator/files/action/so-snyk-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-snyk:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete snyk indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-snyk.*|so-snyk.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-sonicwall-delete.yml b/salt/curator/files/action/so-sonicwall-delete.yml new file mode 100644 index 000000000..c7d38361f --- /dev/null +++ b/salt/curator/files/action/so-sonicwall-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-sonicwall:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete sonicwall indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-sonicwall.*|so-sonicwall.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-sophos-delete.yml b/salt/curator/files/action/so-sophos-delete.yml new file mode 100644 index 000000000..433df908a --- /dev/null +++ b/salt/curator/files/action/so-sophos-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-sophos:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete sophos indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-sophos.*|so-sophos.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-tomcat-delete.yml b/salt/curator/files/action/so-tomcat-delete.yml new file mode 100644 index 000000000..77035613f --- /dev/null +++ b/salt/curator/files/action/so-tomcat-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-tomcat:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete tomcat indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-tomcat.*|so-tomcat.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-zscaler-delete.yml b/salt/curator/files/action/so-zscaler-delete.yml new file mode 100644 index 000000000..238fea083 --- /dev/null +++ b/salt/curator/files/action/so-zscaler-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-zscaler:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete zscaler indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-zscaler.*|so-zscaler.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file From c09cebbd6b9f46334f70b633166e99620f2f19f8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 13:42:11 -0400 Subject: [PATCH 16/61] Add Actions for close and delete in cluster mode --- .../files/bin/so-curator-cluster-close | 44 +++++++++++++++++++ .../files/bin/so-curator-cluster-delete | 44 +++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 salt/curator/files/bin/so-curator-cluster-close create mode 100644 salt/curator/files/bin/so-curator-cluster-delete diff --git a/salt/curator/files/bin/so-curator-cluster-close b/salt/curator/files/bin/so-curator-cluster-close new file mode 100644 index 000000000..1a4124b53 --- /dev/null +++ b/salt/curator/files/bin/so-curator-cluster-close @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +APP=close +lf=/tmp/$APP-pidLockFile +# create empty lock file if none exists +cat /dev/null >> $lf +read lastPID < $lf +# if lastPID is not null and a process with that pid exists , exit +[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit +echo $$ > $lf + +{% from 'filebeat/map.jinja' import THIRDPARTY with context %} +{% from 'filebeat/map.jinja' import SO with context %} + +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-firewall-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ids-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-import-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-osquery-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-close.yml > /dev/null 2>&1; +{% for INDEX in THIRDPARTY -%} +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1; +{% done -%} +{% for INDEX in SO -%} +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} +{% done -%} \ No newline at end of file diff --git a/salt/curator/files/bin/so-curator-cluster-delete b/salt/curator/files/bin/so-curator-cluster-delete new file mode 100644 index 000000000..cccb270b0 --- /dev/null +++ b/salt/curator/files/bin/so-curator-cluster-delete @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +APP=delete +lf=/tmp/$APP-pidLockFile +# create empty lock file if none exists +cat /dev/null >> $lf +read lastPID < $lf +# if lastPID is not null and a process with that pid exists , exit +[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit +echo $$ > $lf + +{% from 'filebeat/map.jinja' import THIRDPARTY with context %} +{% from 'filebeat/map.jinja' import SO with context %} + +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-firewall-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ids-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-import-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-osquery-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-delete.yml > /dev/null 2>&1; +{% for INDEX in THIRDPARTY -%} +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-delete.yml > /dev/null 2>&1; +{% done -%} +{% for INDEX in SO -%} +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-delete.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} +{% done -%} \ No newline at end of file From 476ecccbc10c8b0a414e2c1d060821eeccd6257d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 15:08:10 -0400 Subject: [PATCH 17/61] Add Actions for warm indices --- salt/curator/files/action/so-aws-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-azure-warm.yml | 24 +++++++++++++++ .../files/action/so-barracuda-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-beats-warm.yml | 24 +++++++++++++++ .../curator/files/action/so-bluecoat-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-cef-warm.yml | 24 +++++++++++++++ .../files/action/so-checkpoint-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-cisco-warm.yml | 24 +++++++++++++++ .../curator/files/action/so-cyberark-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-cylance-warm.yml | 24 +++++++++++++++ .../files/action/so-elasticsearch-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-f5-warm.yml | 24 +++++++++++++++ .../curator/files/action/so-firewall-warm.yml | 24 +++++++++++++++ .../curator/files/action/so-fortinet-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-gcp-warm.yml | 24 +++++++++++++++ ...ace.yml => so-google_workspace-delete.yml} | 0 .../files/action/so-google_workspace-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-ids-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-imperva-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-import-warm.yml | 24 +++++++++++++++ .../curator/files/action/so-infoblox-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-juniper-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-kibana-warm.yml | 24 +++++++++++++++ .../curator/files/action/so-logstash-warm.yml | 24 +++++++++++++++ .../files/action/so-microsoft-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-misp-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-netflow-warm.yml | 24 +++++++++++++++ .../files/action/so-netscout-delete.yml | 29 +++++++++++++++++++ .../curator/files/action/so-netscout-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-okta-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-osquery-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-ossec-warm.yml | 24 +++++++++++++++ .../files/action/so-proofpoint-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-radware-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-redis-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-snort-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-snyk-warm.yml | 24 +++++++++++++++ .../files/action/so-sonicwall-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-sophos-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-strelka-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-syslog-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-tomcat-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-zeek-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-zscaler-warm.yml | 24 +++++++++++++++ salt/curator/files/action/warm.yml | 23 --------------- salt/curator/files/bin/so-cluster-warm | 27 +++++++++++++++++ 46 files changed, 1064 insertions(+), 23 deletions(-) create mode 100644 salt/curator/files/action/so-aws-warm.yml create mode 100644 salt/curator/files/action/so-azure-warm.yml create mode 100644 salt/curator/files/action/so-barracuda-warm.yml create mode 100644 salt/curator/files/action/so-beats-warm.yml create mode 100644 salt/curator/files/action/so-bluecoat-warm.yml create mode 100644 salt/curator/files/action/so-cef-warm.yml create mode 100644 salt/curator/files/action/so-checkpoint-warm.yml create mode 100644 salt/curator/files/action/so-cisco-warm.yml create mode 100644 salt/curator/files/action/so-cyberark-warm.yml create mode 100644 salt/curator/files/action/so-cylance-warm.yml create mode 100644 salt/curator/files/action/so-elasticsearch-warm.yml create mode 100644 salt/curator/files/action/so-f5-warm.yml create mode 100644 salt/curator/files/action/so-firewall-warm.yml create mode 100644 salt/curator/files/action/so-fortinet-warm.yml create mode 100644 salt/curator/files/action/so-gcp-warm.yml rename salt/curator/files/action/{so-google_workspace.yml => so-google_workspace-delete.yml} (100%) create mode 100644 salt/curator/files/action/so-google_workspace-warm.yml create mode 100644 salt/curator/files/action/so-ids-warm.yml create mode 100644 salt/curator/files/action/so-imperva-warm.yml create mode 100644 salt/curator/files/action/so-import-warm.yml create mode 100644 salt/curator/files/action/so-infoblox-warm.yml create mode 100644 salt/curator/files/action/so-juniper-warm.yml create mode 100644 salt/curator/files/action/so-kibana-warm.yml create mode 100644 salt/curator/files/action/so-logstash-warm.yml create mode 100644 salt/curator/files/action/so-microsoft-warm.yml create mode 100644 salt/curator/files/action/so-misp-warm.yml create mode 100644 salt/curator/files/action/so-netflow-warm.yml create mode 100644 salt/curator/files/action/so-netscout-delete.yml create mode 100644 salt/curator/files/action/so-netscout-warm.yml create mode 100644 salt/curator/files/action/so-okta-warm.yml create mode 100644 salt/curator/files/action/so-osquery-warm.yml create mode 100644 salt/curator/files/action/so-ossec-warm.yml create mode 100644 salt/curator/files/action/so-proofpoint-warm.yml create mode 100644 salt/curator/files/action/so-radware-warm.yml create mode 100644 salt/curator/files/action/so-redis-warm.yml create mode 100644 salt/curator/files/action/so-snort-warm.yml create mode 100644 salt/curator/files/action/so-snyk-warm.yml create mode 100644 salt/curator/files/action/so-sonicwall-warm.yml create mode 100644 salt/curator/files/action/so-sophos-warm.yml create mode 100644 salt/curator/files/action/so-strelka-warm.yml create mode 100644 salt/curator/files/action/so-syslog-warm.yml create mode 100644 salt/curator/files/action/so-tomcat-warm.yml create mode 100644 salt/curator/files/action/so-zeek-warm.yml create mode 100644 salt/curator/files/action/so-zscaler-warm.yml delete mode 100644 salt/curator/files/action/warm.yml create mode 100644 salt/curator/files/bin/so-cluster-warm diff --git a/salt/curator/files/action/so-aws-warm.yml b/salt/curator/files/action/so-aws-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-aws-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-azure-warm.yml b/salt/curator/files/action/so-azure-warm.yml new file mode 100644 index 000000000..d6f606125 --- /dev/null +++ b/salt/curator/files/action/so-azure-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-azure:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-azure + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-barracuda-warm.yml b/salt/curator/files/action/so-barracuda-warm.yml new file mode 100644 index 000000000..334a4114a --- /dev/null +++ b/salt/curator/files/action/so-barracuda-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-barracuda:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-barracuda + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-beats-warm.yml b/salt/curator/files/action/so-beats-warm.yml new file mode 100644 index 000000000..da9f76656 --- /dev/null +++ b/salt/curator/files/action/so-beats-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-beats:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-beats + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-bluecoat-warm.yml b/salt/curator/files/action/so-bluecoat-warm.yml new file mode 100644 index 000000000..47a8d712f --- /dev/null +++ b/salt/curator/files/action/so-bluecoat-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-bluecoat:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-bluecoat + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-cef-warm.yml b/salt/curator/files/action/so-cef-warm.yml new file mode 100644 index 000000000..0a79fd2ba --- /dev/null +++ b/salt/curator/files/action/so-cef-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-cef:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-cef + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-checkpoint-warm.yml b/salt/curator/files/action/so-checkpoint-warm.yml new file mode 100644 index 000000000..0aaec1e19 --- /dev/null +++ b/salt/curator/files/action/so-checkpoint-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-checkpoint:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-checkpoint + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-cisco-warm.yml b/salt/curator/files/action/so-cisco-warm.yml new file mode 100644 index 000000000..a143a95c2 --- /dev/null +++ b/salt/curator/files/action/so-cisco-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-cisco:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-cisco + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-cyberark-warm.yml b/salt/curator/files/action/so-cyberark-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-cyberark-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-cylance-warm.yml b/salt/curator/files/action/so-cylance-warm.yml new file mode 100644 index 000000000..c9da7e68a --- /dev/null +++ b/salt/curator/files/action/so-cylance-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-cylance:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-cylance + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-elasticsearch-warm.yml b/salt/curator/files/action/so-elasticsearch-warm.yml new file mode 100644 index 000000000..9d82fc27b --- /dev/null +++ b/salt/curator/files/action/so-elasticsearch-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-elasticsearch:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-elasticsearch + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-f5-warm.yml b/salt/curator/files/action/so-f5-warm.yml new file mode 100644 index 000000000..12fbbe7ad --- /dev/null +++ b/salt/curator/files/action/so-f5-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-f5:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-f5 + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-firewall-warm.yml b/salt/curator/files/action/so-firewall-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-firewall-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-fortinet-warm.yml b/salt/curator/files/action/so-fortinet-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-fortinet-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-gcp-warm.yml b/salt/curator/files/action/so-gcp-warm.yml new file mode 100644 index 000000000..3bb9eee80 --- /dev/null +++ b/salt/curator/files/action/so-gcp-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-gcp:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-gcp + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-google_workspace.yml b/salt/curator/files/action/so-google_workspace-delete.yml similarity index 100% rename from salt/curator/files/action/so-google_workspace.yml rename to salt/curator/files/action/so-google_workspace-delete.yml diff --git a/salt/curator/files/action/so-google_workspace-warm.yml b/salt/curator/files/action/so-google_workspace-warm.yml new file mode 100644 index 000000000..7eb2d883f --- /dev/null +++ b/salt/curator/files/action/so-google_workspace-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-google_workspace:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-google_workspace + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-ids-warm.yml b/salt/curator/files/action/so-ids-warm.yml new file mode 100644 index 000000000..0edad5b5b --- /dev/null +++ b/salt/curator/files/action/so-ids-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-ids:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-ids + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-imperva-warm.yml b/salt/curator/files/action/so-imperva-warm.yml new file mode 100644 index 000000000..0297d5cd6 --- /dev/null +++ b/salt/curator/files/action/so-imperva-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-imperva:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-imperva + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-import-warm.yml b/salt/curator/files/action/so-import-warm.yml new file mode 100644 index 000000000..3a6fa3d3d --- /dev/null +++ b/salt/curator/files/action/so-import-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-import:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-import + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-infoblox-warm.yml b/salt/curator/files/action/so-infoblox-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-infoblox-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-juniper-warm.yml b/salt/curator/files/action/so-juniper-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-juniper-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-kibana-warm.yml b/salt/curator/files/action/so-kibana-warm.yml new file mode 100644 index 000000000..b5674c8c3 --- /dev/null +++ b/salt/curator/files/action/so-kibana-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-kibana:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-kibana + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-logstash-warm.yml b/salt/curator/files/action/so-logstash-warm.yml new file mode 100644 index 000000000..826bf2975 --- /dev/null +++ b/salt/curator/files/action/so-logstash-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-logstash:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-logstash + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-microsoft-warm.yml b/salt/curator/files/action/so-microsoft-warm.yml new file mode 100644 index 000000000..551d0cb56 --- /dev/null +++ b/salt/curator/files/action/so-microsoft-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-microsoft:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-microsoft + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-misp-warm.yml b/salt/curator/files/action/so-misp-warm.yml new file mode 100644 index 000000000..af29975b0 --- /dev/null +++ b/salt/curator/files/action/so-misp-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-misp:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-misp + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-netflow-warm.yml b/salt/curator/files/action/so-netflow-warm.yml new file mode 100644 index 000000000..ea57bb72b --- /dev/null +++ b/salt/curator/files/action/so-netflow-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-netflow:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-netflow + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-netscout-delete.yml b/salt/curator/files/action/so-netscout-delete.yml new file mode 100644 index 000000000..3c0e249b5 --- /dev/null +++ b/salt/curator/files/action/so-netscout-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-netscout:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete netscout indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-netscout.*|so-netscout.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-netscout-warm.yml b/salt/curator/files/action/so-netscout-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-netscout-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-okta-warm.yml b/salt/curator/files/action/so-okta-warm.yml new file mode 100644 index 000000000..75764860d --- /dev/null +++ b/salt/curator/files/action/so-okta-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-okta:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-okta + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-osquery-warm.yml b/salt/curator/files/action/so-osquery-warm.yml new file mode 100644 index 000000000..156a83c7a --- /dev/null +++ b/salt/curator/files/action/so-osquery-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-osquery:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-osquery + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-ossec-warm.yml b/salt/curator/files/action/so-ossec-warm.yml new file mode 100644 index 000000000..769d6cbea --- /dev/null +++ b/salt/curator/files/action/so-ossec-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-ossec:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-ossec + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-proofpoint-warm.yml b/salt/curator/files/action/so-proofpoint-warm.yml new file mode 100644 index 000000000..8304ae41a --- /dev/null +++ b/salt/curator/files/action/so-proofpoint-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-proofpoint:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-proofpoint + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-radware-warm.yml b/salt/curator/files/action/so-radware-warm.yml new file mode 100644 index 000000000..8d4337aaf --- /dev/null +++ b/salt/curator/files/action/so-radware-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-radware:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-radware + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-redis-warm.yml b/salt/curator/files/action/so-redis-warm.yml new file mode 100644 index 000000000..a5b1055c3 --- /dev/null +++ b/salt/curator/files/action/so-redis-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-redis:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-redis + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-snort-warm.yml b/salt/curator/files/action/so-snort-warm.yml new file mode 100644 index 000000000..3bbc977e2 --- /dev/null +++ b/salt/curator/files/action/so-snort-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-snort:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-snort + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-snyk-warm.yml b/salt/curator/files/action/so-snyk-warm.yml new file mode 100644 index 000000000..f9b10bbdd --- /dev/null +++ b/salt/curator/files/action/so-snyk-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-snyk:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-snyk + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-sonicwall-warm.yml b/salt/curator/files/action/so-sonicwall-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-sonicwall-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-sophos-warm.yml b/salt/curator/files/action/so-sophos-warm.yml new file mode 100644 index 000000000..40cc60084 --- /dev/null +++ b/salt/curator/files/action/so-sophos-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-sophos:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-sophos + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-strelka-warm.yml b/salt/curator/files/action/so-strelka-warm.yml new file mode 100644 index 000000000..cfa88b0c1 --- /dev/null +++ b/salt/curator/files/action/so-strelka-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-strelka:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-strelka + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-syslog-warm.yml b/salt/curator/files/action/so-syslog-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-syslog-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-tomcat-warm.yml b/salt/curator/files/action/so-tomcat-warm.yml new file mode 100644 index 000000000..8fb7884c0 --- /dev/null +++ b/salt/curator/files/action/so-tomcat-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-tomcat:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-tomcat + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-zeek-warm.yml b/salt/curator/files/action/so-zeek-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-zeek-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-zscaler-warm.yml b/salt/curator/files/action/so-zscaler-warm.yml new file mode 100644 index 000000000..8a7d8187a --- /dev/null +++ b/salt/curator/files/action/so-zscaler-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-zscaler:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-zscaler + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/warm.yml b/salt/curator/files/action/warm.yml deleted file mode 100644 index efa2196b2..000000000 --- a/salt/curator/files/action/warm.yml +++ /dev/null @@ -1,23 +0,0 @@ -#actions: -# 1: -# action: allocation -# description: "Apply shard allocation filtering rules to the specified indices" -# options: -# key: box_type -# value: warm -# allocation_type: require -# wait_for_completion: true -# timeout_override: -# continue_if_exception: false -# disable_action: false -# filters: -# - filtertype: pattern -# kind: prefix -# value: so- -# - filtertype: age -# source: name -# direction: older -# timestring: '%Y.%m.%d' -# unit: days -# unit_count: 3 - diff --git a/salt/curator/files/bin/so-cluster-warm b/salt/curator/files/bin/so-cluster-warm new file mode 100644 index 000000000..c8234c7c9 --- /dev/null +++ b/salt/curator/files/bin/so-cluster-warm @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +APP=warm +lf=/tmp/$APP-pidLockFile +# create empty lock file if none exists +cat /dev/null >> $lf +read lastPID < $lf +# if lastPID is not null and a process with that pid exists , exit +[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit +echo $$ > $lf + +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/warm.yml > /dev/null 2>&1 \ No newline at end of file From ed116cf850bcbfaffcfd4b1aa04e96828781c5ad Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 15:09:26 -0400 Subject: [PATCH 18/61] Add Actions for warm indices --- salt/curator/files/action/so-cyberark-warm.yml | 4 ++-- salt/curator/files/action/so-firewall-warm.yml | 4 ++-- salt/curator/files/action/so-fortinet-warm.yml | 4 ++-- salt/curator/files/action/so-netscout-warm.yml | 4 ++-- salt/curator/files/action/so-sonicwall-warm.yml | 4 ++-- salt/curator/files/action/so-syslog-warm.yml | 4 ++-- salt/curator/files/action/so-zeek-warm.yml | 4 ++-- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/salt/curator/files/action/so-cyberark-warm.yml b/salt/curator/files/action/so-cyberark-warm.yml index 5369ed9a9..8eae0b542 100644 --- a/salt/curator/files/action/so-cyberark-warm.yml +++ b/salt/curator/files/action/so-cyberark-warm.yml @@ -1,4 +1,4 @@ -{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-cyberark:warm', 7) -%} actions: 1: action: allocation @@ -14,7 +14,7 @@ actions: filters: - filtertype: pattern kind: prefix - value: so-aws + value: so-cyberark - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-firewall-warm.yml b/salt/curator/files/action/so-firewall-warm.yml index 5369ed9a9..2e9643dc3 100644 --- a/salt/curator/files/action/so-firewall-warm.yml +++ b/salt/curator/files/action/so-firewall-warm.yml @@ -1,4 +1,4 @@ -{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-firewall:warm', 7) -%} actions: 1: action: allocation @@ -14,7 +14,7 @@ actions: filters: - filtertype: pattern kind: prefix - value: so-aws + value: so-firewall - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-fortinet-warm.yml b/salt/curator/files/action/so-fortinet-warm.yml index 5369ed9a9..db9a6f2db 100644 --- a/salt/curator/files/action/so-fortinet-warm.yml +++ b/salt/curator/files/action/so-fortinet-warm.yml @@ -1,4 +1,4 @@ -{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-fortinet:warm', 7) -%} actions: 1: action: allocation @@ -14,7 +14,7 @@ actions: filters: - filtertype: pattern kind: prefix - value: so-aws + value: so-fortinet - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-netscout-warm.yml b/salt/curator/files/action/so-netscout-warm.yml index 5369ed9a9..1b93c3118 100644 --- a/salt/curator/files/action/so-netscout-warm.yml +++ b/salt/curator/files/action/so-netscout-warm.yml @@ -1,4 +1,4 @@ -{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-netscout:warm', 7) -%} actions: 1: action: allocation @@ -14,7 +14,7 @@ actions: filters: - filtertype: pattern kind: prefix - value: so-aws + value: so-netscout - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-sonicwall-warm.yml b/salt/curator/files/action/so-sonicwall-warm.yml index 5369ed9a9..fa8ceb3e4 100644 --- a/salt/curator/files/action/so-sonicwall-warm.yml +++ b/salt/curator/files/action/so-sonicwall-warm.yml @@ -1,4 +1,4 @@ -{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-sonicwall:warm', 7) -%} actions: 1: action: allocation @@ -14,7 +14,7 @@ actions: filters: - filtertype: pattern kind: prefix - value: so-aws + value: so-sonicwall - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-syslog-warm.yml b/salt/curator/files/action/so-syslog-warm.yml index 5369ed9a9..e5ebb2fa6 100644 --- a/salt/curator/files/action/so-syslog-warm.yml +++ b/salt/curator/files/action/so-syslog-warm.yml @@ -1,4 +1,4 @@ -{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-syslog:warm', 7) -%} actions: 1: action: allocation @@ -14,7 +14,7 @@ actions: filters: - filtertype: pattern kind: prefix - value: so-aws + value: so-syslog - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-zeek-warm.yml b/salt/curator/files/action/so-zeek-warm.yml index 5369ed9a9..2b4b6a729 100644 --- a/salt/curator/files/action/so-zeek-warm.yml +++ b/salt/curator/files/action/so-zeek-warm.yml @@ -1,4 +1,4 @@ -{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:warm', 7) -%} actions: 1: action: allocation @@ -14,7 +14,7 @@ actions: filters: - filtertype: pattern kind: prefix - value: so-aws + value: so-zeek - filtertype: age source: name direction: older From f6f4375e1396956124d875388e6176a8fb2873da Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 15:34:26 -0400 Subject: [PATCH 19/61] Add logic for cronjobs --- ...o-cluster-warm => so-curator-cluster-warm} | 0 salt/curator/init.sls | 127 ++++++++++++------ 2 files changed, 85 insertions(+), 42 deletions(-) rename salt/curator/files/bin/{so-cluster-warm => so-curator-cluster-warm} (100%) diff --git a/salt/curator/files/bin/so-cluster-warm b/salt/curator/files/bin/so-curator-cluster-warm similarity index 100% rename from salt/curator/files/bin/so-cluster-warm rename to salt/curator/files/bin/so-curator-cluster-warm diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 1acea5d25..57da8bfa1 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -5,6 +5,9 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} {% set REMOVECURATORCRON = False %} +{% set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %} +{% set HOTWARM = salt['pillar.get']('elasticsearch:hot_warm_enabled', False) %} + {% if grains['role'] in ['so-eval', 'so-node', 'so-managersearch', 'so-heavynode', 'so-standalone', 'so-manager'] %} {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} {% from "curator/map.jinja" import CURATOROPTIONS with context %} @@ -148,62 +151,102 @@ delete_so-curator_so-status: {% if REMOVECURATORCRON %} so-curatorcloseddeletecron: - cron.absent: - - name: /usr/sbin/so-curator-closed-delete > /opt/so/log/curator/cron-closed-delete.log 2>&1 - - user: root + cron.absent: + - name: /usr/sbin/so-curator-closed-delete > /opt/so/log/curator/cron-closed-delete.log 2>&1 + - user: root so-curatorclosecron: - cron.absent: - - name: /usr/sbin/so-curator-close > /opt/so/log/curator/cron-close.log 2>&1 - - user: root + cron.absent: + - name: /usr/sbin/so-curator-close > /opt/so/log/curator/cron-close.log 2>&1 + - user: root so-curatordeletecron: - cron.absent: - - name: /usr/sbin/so-curator-delete > /opt/so/log/curator/cron-delete.log 2>&1 - - user: root + cron.absent: + - name: /usr/sbin/so-curator-delete > /opt/so/log/curator/cron-delete.log 2>&1 + - user: root {% else %} + {% if TRUECLUSTER is sameas true %} so-curatorcloseddeletecron: - cron.present: - - name: /usr/sbin/so-curator-closed-delete > /opt/so/log/curator/cron-closed-delete.log 2>&1 - - user: root - - minute: '*' - - hour: '*' - - daymonth: '*' - - month: '*' - - dayweek: '*' + cron.absent: + - name: /usr/sbin/so-curator-closed-delete > /opt/so/log/curator/cron-closed-delete.log 2>&1 + - user: root so-curatorclosecron: - cron.present: - - name: /usr/sbin/so-curator-close > /opt/so/log/curator/cron-close.log 2>&1 - - user: root - - minute: '*' - - hour: '*' - - daymonth: '*' - - month: '*' - - dayweek: '*' + cron.absent: + - name: /usr/sbin/so-curator-close > /opt/so/log/curator/cron-close.log 2>&1 + - user: root so-curatordeletecron: - cron.present: - - name: /usr/sbin/so-curator-delete > /opt/so/log/curator/cron-delete.log 2>&1 - - user: root - - minute: '*' - - hour: '*' - - daymonth: '*' - - month: '*' - - dayweek: '*' + cron.absent: + - name: /usr/sbin/so-curator-delete > /opt/so/log/curator/cron-delete.log 2>&1 + - user: root + +so-curatorclusterclose: + cron.present: + - name: /usr/sbin/so-curator-cluster-close > /opt/so/log/curator/cron-close.log 2>&1 + - user: root + - minute: '*' + - hour: '*/1' + - daymonth: '*' + - month: '*' + - dayweek: '*' +so-curatorclusterdelete: + cron.present: + - name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-close.log 2>&1 + - user: root + - minute: '*' + - hour: '*/1' + - daymonth: '*' + - month: '*' + - dayweek: '*' + {% if HOTWARM is sameas true %} +so-curatorclusterwarm: + cron.present: + - name: /usr/sbin/so-curator-cluster-warm > /opt/so/log/curator/cron-close.log 2>&1 + - user: root + - minute: '*' + - hour: '*/1' + - daymonth: '*' + - month: '*' + - dayweek: '*' + {% endif %%} + + {% else %} +so-curatorcloseddeletecron: + cron.present: + - name: /usr/sbin/so-curator-closed-delete > /opt/so/log/curator/cron-closed-delete.log 2>&1 + - user: root + - minute: '*/5' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' + +so-curatorclosecron: + cron.present: + - name: /usr/sbin/so-curator-close > /opt/so/log/curator/cron-close.log 2>&1 + - user: root + - minute: '*/5' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' + +so-curatordeletecron: + cron.present: + - name: /usr/sbin/so-curator-delete > /opt/so/log/curator/cron-delete.log 2>&1 + - user: root + - minute: '*/5' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' + + {% endif %} {% endif %} - -# Begin Curator Cron Jobs - -# Close -# Delete -# Hot Warm -# Segment Merge - -# End Curator Cron Jobs {% endif %} {% else %} From 9f51f02ab48538a014b866012991da26ed953e4f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 15:40:09 -0400 Subject: [PATCH 20/61] Add logic for cronjobs --- salt/curator/init.sls | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 57da8bfa1..8900c35d8 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -187,7 +187,7 @@ so-curatorclusterclose: cron.present: - name: /usr/sbin/so-curator-cluster-close > /opt/so/log/curator/cron-close.log 2>&1 - user: root - - minute: '*' + - minute: '2' - hour: '*/1' - daymonth: '*' - month: '*' @@ -197,7 +197,7 @@ so-curatorclusterdelete: cron.present: - name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-close.log 2>&1 - user: root - - minute: '*' + - minute: '2' - hour: '*/1' - daymonth: '*' - month: '*' @@ -207,7 +207,7 @@ so-curatorclusterwarm: cron.present: - name: /usr/sbin/so-curator-cluster-warm > /opt/so/log/curator/cron-close.log 2>&1 - user: root - - minute: '*' + - minute: '2' - hour: '*/1' - daymonth: '*' - month: '*' From dc25ed2594b01430ca23722b6bcd9deb1c01043a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 15:43:48 -0400 Subject: [PATCH 21/61] Add logic for cronjobs --- salt/curator/init.sls | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 8900c35d8..b7b580af7 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -167,22 +167,7 @@ so-curatordeletecron: {% else %} - {% if TRUECLUSTER is sameas true %} -so-curatorcloseddeletecron: - cron.absent: - - name: /usr/sbin/so-curator-closed-delete > /opt/so/log/curator/cron-closed-delete.log 2>&1 - - user: root - -so-curatorclosecron: - cron.absent: - - name: /usr/sbin/so-curator-close > /opt/so/log/curator/cron-close.log 2>&1 - - user: root - -so-curatordeletecron: - cron.absent: - - name: /usr/sbin/so-curator-delete > /opt/so/log/curator/cron-delete.log 2>&1 - - user: root - + {% if TRUECLUSTER is sameas true %} so-curatorclusterclose: cron.present: - name: /usr/sbin/so-curator-cluster-close > /opt/so/log/curator/cron-close.log 2>&1 From c12f380bc348c10b4b1adda9d65f22964f29539f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 15:51:34 -0400 Subject: [PATCH 22/61] Only enable curator on Manager in true cluster --- salt/curator/map.jinja | 4 ++-- salt/top.sls | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/curator/map.jinja b/salt/curator/map.jinja index 470143a4f..bbcff2a9c 100644 --- a/salt/curator/map.jinja +++ b/salt/curator/map.jinja @@ -4,10 +4,10 @@ {% do CURATOROPTIONS.update({'manage_sostatus': True}) %} # don't start the docker container if curator is disabled via pillar -{% if not ENABLED or grains.id.split('_')|last == 'manager'%} +{% if not ENABLED or (TRUECLUSTER and grains.id.split('_')|last == 'searchnode') or (not TRUECLUSTER and grains.id.split('_')|last == 'manager') %} {% do CURATOROPTIONS.update({'start': False}) %} {% do CURATOROPTIONS.update({'status': 'absent'}) %} - {% if grains.id.split('_')|last == 'manager' %} + {% if (TRUECLUSTER and grains.id.split('_')|last == 'searchnode') or (not TRUECLUSTER and grains.id.split('_')|last == 'manager') %} {% do CURATOROPTIONS.update({'manage_sostatus': False}) %} {% endif %} {% else %} diff --git a/salt/top.sls b/salt/top.sls index b38d315d8..e2c980e1d 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -189,6 +189,7 @@ base: {%- if KIBANA %} - kibana {%- endif %} + - curator {%- if ELASTALERT %} - elastalert {%- endif %} From 5b9ef3bc0d2ed5b8881398ebe89a45b7019e9fd2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 15:55:44 -0400 Subject: [PATCH 23/61] Only enable curator on Manager in true cluster --- salt/curator/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/curator/init.sls b/salt/curator/init.sls index b7b580af7..10cffe680 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -197,7 +197,7 @@ so-curatorclusterwarm: - daymonth: '*' - month: '*' - dayweek: '*' - {% endif %%} + {% endif %} {% else %} so-curatorcloseddeletecron: From 0c8e11dc9f3903b8cd51e26642d217aa15bd7eb9 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 16:13:05 -0400 Subject: [PATCH 24/61] Only enable curator on Manager in true cluster --- salt/curator/init.sls | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 10cffe680..95e62ac95 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -92,6 +92,30 @@ curdel: - group: 939 - mode: 755 +curclusterclose: + file.managed: + - name: /usr/sbin/so-curator-cluster-close + - source: salt://curator/files/bin/so-curator-cluster-close + - user: 934 + - group: 939 + - mode: 755 + +curclusterdelete: + file.managed: + - name: /usr/sbin/so-curator-cluster-delete + - source: salt://curator/files/bin/so-curator-cluster-delete + - user: 934 + - group: 939 + - mode: 755 + +curclustercwarm: + file.managed: + - name: /usr/sbin/so-curator-cluster-warm + - source: salt://curator/files/bin/so-curator-cluster-warm + - user: 934 + - group: 939 + - mode: 755 + so-curator: docker_container.{{ CURATOROPTIONS.status }}: {% if CURATOROPTIONS.status == 'running' %} From 958575c22a2fdbe4f09910c65960305fa1561b73 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 16:17:55 -0400 Subject: [PATCH 25/61] Only enable curator on Manager in true cluster --- salt/curator/init.sls | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 95e62ac95..54fcff3a6 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -99,6 +99,7 @@ curclusterclose: - user: 934 - group: 939 - mode: 755 + - template: jinja curclusterdelete: file.managed: @@ -107,6 +108,7 @@ curclusterdelete: - user: 934 - group: 939 - mode: 755 + - template: jinja curclustercwarm: file.managed: @@ -115,7 +117,8 @@ curclustercwarm: - user: 934 - group: 939 - mode: 755 - + - template: jinja + so-curator: docker_container.{{ CURATOROPTIONS.status }}: {% if CURATOROPTIONS.status == 'running' %} From 62edfd0b7fde61491cf75c5ec499e1ca5ada16ec Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 16:20:42 -0400 Subject: [PATCH 26/61] Only enable curator on Manager in true cluster --- salt/curator/files/bin/so-curator-cluster-close | 4 ++-- salt/curator/files/bin/so-curator-cluster-delete | 4 ++-- salt/curator/files/bin/so-curator-cluster-warm | 16 +++++++++++++++- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/salt/curator/files/bin/so-curator-cluster-close b/salt/curator/files/bin/so-curator-cluster-close index 1a4124b53..b2043d3b6 100644 --- a/salt/curator/files/bin/so-curator-cluster-close +++ b/salt/curator/files/bin/so-curator-cluster-close @@ -38,7 +38,7 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-close.yml > /dev/null 2>&1; {% for INDEX in THIRDPARTY -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1; -{% done -%} +{% endfor -%} {% for INDEX in SO -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} -{% done -%} \ No newline at end of file +{% endfor -%} \ No newline at end of file diff --git a/salt/curator/files/bin/so-curator-cluster-delete b/salt/curator/files/bin/so-curator-cluster-delete index cccb270b0..df7f2c4bf 100644 --- a/salt/curator/files/bin/so-curator-cluster-delete +++ b/salt/curator/files/bin/so-curator-cluster-delete @@ -38,7 +38,7 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-delete.yml > /dev/null 2>&1; {% for INDEX in THIRDPARTY -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-delete.yml > /dev/null 2>&1; -{% done -%} +{% endfor -%} {% for INDEX in SO -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-delete.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} -{% done -%} \ No newline at end of file +{% endfor -%} \ No newline at end of file diff --git a/salt/curator/files/bin/so-curator-cluster-warm b/salt/curator/files/bin/so-curator-cluster-warm index c8234c7c9..db54b7fe9 100644 --- a/salt/curator/files/bin/so-curator-cluster-warm +++ b/salt/curator/files/bin/so-curator-cluster-warm @@ -24,4 +24,18 @@ read lastPID < $lf [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit echo $$ > $lf -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/warm.yml > /dev/null 2>&1 \ No newline at end of file +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-warm.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-warm.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-firewall-warm.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ids-warm.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-import-warm.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-osquery-warm.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-warm.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-warm.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-warm.yml > /dev/null 2>&1; +{% for INDEX in THIRDPARTY -%} +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-warm.yml > /dev/null 2>&1; +{% endfor -%} +{% for INDEX in SO -%} +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-warm.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} +{% endfor -%} \ No newline at end of file From e7b04b862fb2eb21b358fa6e20d682db6d0404aa Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 16:21:48 -0400 Subject: [PATCH 27/61] Only enable curator on Manager in true cluster --- salt/curator/files/bin/so-curator-cluster-warm | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/curator/files/bin/so-curator-cluster-warm b/salt/curator/files/bin/so-curator-cluster-warm index db54b7fe9..ec08a31c0 100644 --- a/salt/curator/files/bin/so-curator-cluster-warm +++ b/salt/curator/files/bin/so-curator-cluster-warm @@ -24,6 +24,9 @@ read lastPID < $lf [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit echo $$ > $lf +{% from 'filebeat/map.jinja' import THIRDPARTY with context %} +{% from 'filebeat/map.jinja' import SO with context %} + docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-warm.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-warm.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-firewall-warm.yml > /dev/null 2>&1; From e09aa4e5d4b359cd7885f58df003429bae495912 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 16:35:19 -0400 Subject: [PATCH 28/61] Only enable curator on Manager in true cluster --- salt/curator/files/bin/so-curator-cluster-close | 4 ++-- salt/curator/files/bin/so-curator-cluster-delete | 4 ++-- salt/curator/files/bin/so-curator-cluster-warm | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/curator/files/bin/so-curator-cluster-close b/salt/curator/files/bin/so-curator-cluster-close index b2043d3b6..7b8c41c9e 100644 --- a/salt/curator/files/bin/so-curator-cluster-close +++ b/salt/curator/files/bin/so-curator-cluster-close @@ -36,9 +36,9 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-close.yml > /dev/null 2>&1; -{% for INDEX in THIRDPARTY -%} +{% for INDEX in THIRDPARTY.modules.keys() -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1; {% endfor -%} -{% for INDEX in SO -%} +{% for INDEX in SO.modules.keys() -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} {% endfor -%} \ No newline at end of file diff --git a/salt/curator/files/bin/so-curator-cluster-delete b/salt/curator/files/bin/so-curator-cluster-delete index df7f2c4bf..21195da5d 100644 --- a/salt/curator/files/bin/so-curator-cluster-delete +++ b/salt/curator/files/bin/so-curator-cluster-delete @@ -36,9 +36,9 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-delete.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-delete.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-delete.yml > /dev/null 2>&1; -{% for INDEX in THIRDPARTY -%} +{% for INDEX in THIRDPARTY.modules.keys() -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-delete.yml > /dev/null 2>&1; {% endfor -%} -{% for INDEX in SO -%} +{% for INDEX in SO.modules.keys() -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-delete.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} {% endfor -%} \ No newline at end of file diff --git a/salt/curator/files/bin/so-curator-cluster-warm b/salt/curator/files/bin/so-curator-cluster-warm index ec08a31c0..4348ba2e3 100644 --- a/salt/curator/files/bin/so-curator-cluster-warm +++ b/salt/curator/files/bin/so-curator-cluster-warm @@ -36,9 +36,9 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-warm.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-warm.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-warm.yml > /dev/null 2>&1; -{% for INDEX in THIRDPARTY -%} +{% for INDEX in THIRDPARTY.modules.keys() -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-warm.yml > /dev/null 2>&1; {% endfor -%} -{% for INDEX in SO -%} +{% for INDEX in SO.modules.keys() -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-warm.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} {% endfor -%} \ No newline at end of file From 61ebedc0e9439c43c57c4ae8f2fff118984f16e6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 16:56:08 -0400 Subject: [PATCH 29/61] Only enable curator on Manager in true cluster --- salt/curator/files/action/so-infoblox-warm.yml | 4 ++-- salt/curator/files/action/so-kibana-close.yml | 6 +++--- salt/curator/init.sls | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/salt/curator/files/action/so-infoblox-warm.yml b/salt/curator/files/action/so-infoblox-warm.yml index 5369ed9a9..a2f571b7a 100644 --- a/salt/curator/files/action/so-infoblox-warm.yml +++ b/salt/curator/files/action/so-infoblox-warm.yml @@ -1,4 +1,4 @@ -{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-infoblox:warm', 7) -%} actions: 1: action: allocation @@ -14,7 +14,7 @@ actions: filters: - filtertype: pattern kind: prefix - value: so-aws + value: so-infoblox - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-kibana-close.yml b/salt/curator/files/action/so-kibana-close.yml index 05583d853..7347fb01c 100644 --- a/salt/curator/files/action/so-kibana-close.yml +++ b/salt/curator/files/action/so-kibana-close.yml @@ -1,4 +1,4 @@ -{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-kibana:close', 30) -%} --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" @@ -10,7 +10,7 @@ actions: 1: action: close description: >- - Close IDS indices older than {{cur_close_days}} days. + Close kibana indices older than {{cur_close_days}} days. options: delete_aliases: False timeout_override: @@ -19,7 +19,7 @@ actions: filters: - filtertype: pattern kind: regex - value: '^(logstash-ids.*|so-ids.*)$' + value: '^(logstash-kibana.*|so-kibana.*)$' - filtertype: age source: name direction: older diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 54fcff3a6..87e1fec33 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -207,7 +207,7 @@ so-curatorclusterclose: so-curatorclusterdelete: cron.present: - - name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-close.log 2>&1 + - name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-delete.log 2>&1 - user: root - minute: '2' - hour: '*/1' @@ -217,7 +217,7 @@ so-curatorclusterdelete: {% if HOTWARM is sameas true %} so-curatorclusterwarm: cron.present: - - name: /usr/sbin/so-curator-cluster-warm > /opt/so/log/curator/cron-close.log 2>&1 + - name: /usr/sbin/so-curator-cluster-warm > /opt/so/log/curator/cron-warm.log 2>&1 - user: root - minute: '2' - hour: '*/1' From 527a793e9419ada93a38257e90cf51b97ea90a16 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 16:59:41 -0400 Subject: [PATCH 30/61] Only enable curator on Manager in true cluster --- salt/curator/files/action/so-o365-warm.yml | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 salt/curator/files/action/so-o365-warm.yml diff --git a/salt/curator/files/action/so-o365-warm.yml b/salt/curator/files/action/so-o365-warm.yml new file mode 100644 index 000000000..cbb7bc24e --- /dev/null +++ b/salt/curator/files/action/so-o365-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-o365:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-o365 + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + From 007b15979a11a2297bf0badec2fae50c2ac1e7c6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 1 Sep 2021 09:25:14 -0400 Subject: [PATCH 31/61] Non Cluster honor closed indices values --- salt/curator/files/bin/so-curator-close | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/salt/curator/files/bin/so-curator-close b/salt/curator/files/bin/so-curator-close index 6f7b96790..c1e940b4d 100644 --- a/salt/curator/files/bin/so-curator-close +++ b/salt/curator/files/bin/so-curator-close @@ -23,5 +23,22 @@ read lastPID < $lf # if lastPID is not null and a process with that pid exists , exit [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit echo $$ > $lf +{% from 'filebeat/map.jinja' import THIRDPARTY with context %} +{% from 'filebeat/map.jinja' import SO with context %} -/usr/sbin/so-curator-closed-delete > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-firewall-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ids-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-import-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-osquery-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-close.yml > /dev/null 2>&1 +/usr/sbin/so-curator-closed-delete > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-firewall-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ids-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-import-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-osquery-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-close.yml > /dev/null 2>&1; +{% for INDEX in THIRDPARTY.modules.keys() -%} +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1; +{% endfor -%} +{% for INDEX in SO.modules.keys() -%} +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} +{% endfor -%} From 324e6b12e2bbad13686c172e09559542de810231 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 1 Sep 2021 09:32:32 -0400 Subject: [PATCH 32/61] Add jinja template --- salt/curator/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 87e1fec33..c604571ba 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -83,6 +83,7 @@ curclose: - user: 934 - group: 939 - mode: 755 + - template: jinja curdel: file.managed: From 576c893eb3731817ecf9eb709467e6d6ab640f3a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 1 Sep 2021 15:08:53 -0400 Subject: [PATCH 33/61] Exit on missing file argument --- salt/common/tools/sbin/soup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 60e0ce93f..87fe56784 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -788,6 +788,7 @@ main() { check_airgap if [[ $is_airgap -eq 0 && $UNATTENDED == true && -z $ISOLOC ]]; then echo "Missing file argument for unattended airgap upgrade." + exit 1 fi echo "Checking to see if this is a manager." echo "" From 446821e9fdd657d580787e39835b2d55a368aa58 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 1 Sep 2021 15:11:18 -0400 Subject: [PATCH 34/61] Use exit code 0 when printing error message before exiting soup --- salt/common/tools/sbin/soup | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 87fe56784..de26e73ea 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -127,7 +127,7 @@ EOF if [ ! -f /tmp/soagupdate/SecurityOnion/VERSION ]; then echo "Something went wrong trying to mount the ISO." echo "Ensure you verify the ISO that you downloaded." - exit 1 + exit 0 else echo "ISO has been mounted!" fi @@ -140,14 +140,14 @@ EOF if [ ! -f /tmp/soagupdate/SecurityOnion/VERSION ]; then echo "Something went wrong trying to mount the device." echo "Ensure you verify the ISO that you downloaded." - exit 1 + exit 0 else echo "Device has been mounted!" fi else echo "Could not find Security Onion ISO content at ${ISOLOC}" echo "Ensure the path you entered is correct, and that you verify the ISO that you downloaded." - exit 1 + exit 0 fi fi } @@ -157,7 +157,7 @@ airgap_update_dockers() { # Let's copy the tarball if [[ ! -f $AGDOCKER/registry.tar ]]; then echo "Unable to locate registry. Exiting" - exit 1 + exit 0 else echo "Stopping the registry docker" docker stop so-dockerregistry @@ -631,7 +631,7 @@ upgrade_space() { clean_dockers if ! verify_upgradespace; then echo "There is not enough space to perform the upgrade. Please free up space and try again" - exit 1 + exit 0 fi else echo "You have enough space for upgrade. Proceeding with soup." @@ -788,7 +788,7 @@ main() { check_airgap if [[ $is_airgap -eq 0 && $UNATTENDED == true && -z $ISOLOC ]]; then echo "Missing file argument for unattended airgap upgrade." - exit 1 + exit 0 fi echo "Checking to see if this is a manager." echo "" @@ -885,7 +885,7 @@ main() { echo "Once the issue is resolved, run soup again." echo "Exiting." echo "" - exit 1 + exit 0 else echo "Salt upgrade success." echo "" From 556bad6925f59cb98bba37f88d46bda080856b2d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 1 Sep 2021 15:13:43 -0400 Subject: [PATCH 35/61] Add eventfields for new default logs --- salt/soc/files/soc/hunt.eventfields.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/hunt.eventfields.json b/salt/soc/files/soc/hunt.eventfields.json index e8af03a5c..354aa31ce 100644 --- a/salt/soc/files/soc/hunt.eventfields.json +++ b/salt/soc/files/soc/hunt.eventfields.json @@ -40,5 +40,9 @@ ":strelka:file": ["soc_timestamp", "file.name", "file.size", "hash.md5", "file.source", "file.mime_type", "log.id.fuid" ], ":suricata:": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "rule.name", "rule.category", "event.severity_label", "log.id.uid", "network.community_id" ], ":sysmon:": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "source.hostname", "event.dataset", "process.executable", "user.name" ], - ":windows_eventlog:": ["soc_timestamp", "user.name" ] + ":windows_eventlog:": ["soc_timestamp", "user.name" ], + ":elasticsearch:": ["soc_timestamp", "agent.name", "message", "log.level", "metadata.version", "metadata.pipeline", "event.dataset" ], + ":kibana:": ["soc_timestamp", "host.name", "message", "kibana.log.meta.req.headers.x-real-ip", "event.dataset" ], + "::rootcheck": ["soc_timestamp", "host.name", "metadata.ip_address", "log.full", "event.dataset", "event.module" ], + "::syscollector": ["soc_timestamp", "host.name", "metadata.ip_address", "wazuh.data.type", "event.dataset", "event.module" ] } From 0ad3d826ebf7ea3cccc08c8a392a5e064b1bde1f Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 2 Sep 2021 07:07:36 -0400 Subject: [PATCH 36/61] Invalid password message should also mention that dollar signs are not allowed --- salt/common/tools/sbin/so-fleet-user-add | 2 +- salt/common/tools/sbin/so-fleet-user-update | 2 +- salt/common/tools/sbin/so-thehive-user-add | 2 +- salt/common/tools/sbin/so-thehive-user-update | 2 +- setup/so-whiptail | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/so-fleet-user-add b/salt/common/tools/sbin/so-fleet-user-add index e905424a9..9d80c2076 100755 --- a/salt/common/tools/sbin/so-fleet-user-add +++ b/salt/common/tools/sbin/so-fleet-user-add @@ -42,7 +42,7 @@ fi read -rs FLEET_PASS if ! check_password "$FLEET_PASS"; then - echo "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password." + echo "Password is invalid. Please exclude single quotes, double quotes, dollar signs, and backslashes from the password." exit 2 fi diff --git a/salt/common/tools/sbin/so-fleet-user-update b/salt/common/tools/sbin/so-fleet-user-update index e6a142d1d..793f7b622 100755 --- a/salt/common/tools/sbin/so-fleet-user-update +++ b/salt/common/tools/sbin/so-fleet-user-update @@ -52,7 +52,7 @@ fi read -rs FLEET_PASS if ! check_password "$FLEET_PASS"; then - echo "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password." + echo "Password is invalid. Please exclude single quotes, double quotes, dollar signs, and backslashes from the password." exit 2 fi diff --git a/salt/common/tools/sbin/so-thehive-user-add b/salt/common/tools/sbin/so-thehive-user-add index 5d174fdf1..e26dc58fc 100755 --- a/salt/common/tools/sbin/so-thehive-user-add +++ b/salt/common/tools/sbin/so-thehive-user-add @@ -42,7 +42,7 @@ fi read -rs THEHIVE_PASS if ! check_password "$THEHIVE_PASS"; then - echo "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password." + echo "Password is invalid. Please exclude single quotes, double quotes, dollar signs, and backslashes from the password." exit 2 fi diff --git a/salt/common/tools/sbin/so-thehive-user-update b/salt/common/tools/sbin/so-thehive-user-update index 6df199f6a..fdda5eaa7 100755 --- a/salt/common/tools/sbin/so-thehive-user-update +++ b/salt/common/tools/sbin/so-thehive-user-update @@ -42,7 +42,7 @@ fi read -rs THEHIVE_PASS if ! check_password "$THEHIVE_PASS"; then - echo "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password." + echo "Password is invalid. Please exclude single quotes, double quotes, dollar signs, and backslashes from the password." exit 2 fi diff --git a/setup/so-whiptail b/setup/so-whiptail index 780411841..125335729 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -843,7 +843,7 @@ whiptail_invalid_pass_characters_warning() { [ -n "$TESTING" ] && return - whiptail --title "$whiptail_title" --msgbox "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password." 8 75 + whiptail --title "$whiptail_title" --msgbox "Password is invalid. Please exclude single quotes, double quotes, dollar signs, and backslashes from the password." 8 75 } whiptail_invalid_pass_warning() { From 10126bb7ef05aeed6545dde9a9151faee9b9677e Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 2 Sep 2021 09:44:57 -0400 Subject: [PATCH 37/61] Auth enhancements --- .../tools/sbin/so-elasticsearch-roles-load | 57 ++++++++ salt/common/tools/sbin/so-user | 132 ++++++++++++++---- salt/elasticsearch/init.sls | 29 +++- salt/elasticsearch/roles/analyst.json | 45 ++++++ salt/soc/files/kratos/schema.json | 4 - salt/soc/files/soc/custom_roles | 20 +++ salt/soc/files/soc/soc.json | 8 ++ salt/soc/init.sls | 11 ++ setup/so-functions | 2 +- 9 files changed, 269 insertions(+), 39 deletions(-) create mode 100644 salt/common/tools/sbin/so-elasticsearch-roles-load create mode 100644 salt/elasticsearch/roles/analyst.json create mode 100644 salt/soc/files/soc/custom_roles diff --git a/salt/common/tools/sbin/so-elasticsearch-roles-load b/salt/common/tools/sbin/so-elasticsearch-roles-load new file mode 100644 index 000000000..3b0f580fe --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-roles-load @@ -0,0 +1,57 @@ +#!/bin/bash +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +{%- set mainint = salt['pillar.get']('host:mainint') %} +{%- set MYIP = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %} + +default_conf_dir=/opt/so/conf +ELASTICSEARCH_HOST="{{ MYIP }}" +ELASTICSEARCH_PORT=9200 + +# Define a default directory to load roles from +ELASTICSEARCH_ROLES="$default_conf_dir/elasticsearch/roles/" + +# Wait for ElasticSearch to initialize +echo -n "Waiting for ElasticSearch..." +COUNT=0 +ELASTICSEARCH_CONNECTED="no" +while [[ "$COUNT" -le 240 ]]; do + {{ ELASTICCURL }} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" + if [ $? -eq 0 ]; then + ELASTICSEARCH_CONNECTED="yes" + echo "connected!" + break + else + ((COUNT+=1)) + sleep 1 + echo -n "." + fi +done +if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then + echo + echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" + echo +fi + +cd ${ELASTICSEARCH_ROLES} + +echo "Loading templates..." +for role in *; do + name=$(echo "$role" | cut -d. -f1) + so-elasticsearch-query security/roles/$name -XPUT -d @"$role" +done + +cd - >/dev/null diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 742c3ca5d..7ec094efb 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -18,11 +18,17 @@ source $(dirname $0)/so-common +DEFAULT_ROLE=analyst + if [[ $# -lt 1 || $# -gt 2 ]]; then - echo "Usage: $0 [email]" + echo "Usage: $0 [email] [role]" + echo "" + echo " where is one of the following:" echo "" echo " list: Lists all user email addresses currently defined in the identity system" echo " add: Adds a new user to the identity system; requires 'email' parameter" + echo " addrole: Grants a role to an existing user; requires 'email' and 'role' parameters" + echo " delrole: Removes a role from an existing user; requires 'email' and 'role' parameters" echo " update: Updates a user's password; requires 'email' parameter" echo " enable: Enables a user; requires 'email' parameter" echo " disable: Disables a user; requires 'email' parameter" @@ -36,6 +42,7 @@ fi operation=$1 email=$2 +role=$3 kratosUrl=${KRATOS_URL:-http://127.0.0.1:4434} databasePath=${KRATOS_DB_PATH:-/opt/so/conf/kratos/db/db.sqlite} @@ -138,10 +145,9 @@ function updatePassword() { function createElasticFile() { filename=$1 - tmpFile=${filename} - truncate -s 0 "$tmpFile" - chmod 600 "$tmpFile" - chown "${esUID}:${esGID}" "$tmpFile" + truncate -s 0 "$filename" + chmod 600 "$filename" + chown "${esUID}:${esGID}" "$filename" } function syncElasticSystemUser() { @@ -174,28 +180,15 @@ function syncElasticSystemRole() { function syncElastic() { echo "Syncing users between SOC and Elastic..." usersTmpFile="${elasticUsersFile}.tmp" - rolesTmpFile="${elasticRolesFile}.tmp" createElasticFile "${usersTmpFile}" - createElasticFile "${rolesTmpFile}" authPillarJson=$(lookup_salt_value "auth" "elasticsearch" "pillar" "json") syncElasticSystemUser "$authPillarJson" "so_elastic_user" "$usersTmpFile" - syncElasticSystemRole "$authPillarJson" "so_elastic_user" "superuser" "$rolesTmpFile" - syncElasticSystemUser "$authPillarJson" "so_kibana_user" "$usersTmpFile" - syncElasticSystemRole "$authPillarJson" "so_kibana_user" "superuser" "$rolesTmpFile" - syncElasticSystemUser "$authPillarJson" "so_logstash_user" "$usersTmpFile" - syncElasticSystemRole "$authPillarJson" "so_logstash_user" "superuser" "$rolesTmpFile" - syncElasticSystemUser "$authPillarJson" "so_beats_user" "$usersTmpFile" - syncElasticSystemRole "$authPillarJson" "so_beats_user" "superuser" "$rolesTmpFile" - syncElasticSystemUser "$authPillarJson" "so_monitor_user" "$usersTmpFile" - syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_collector" "$rolesTmpFile" - syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_agent" "$rolesTmpFile" - syncElasticSystemRole "$authPillarJson" "so_monitor_user" "monitoring_user" "$rolesTmpFile" if [[ -f "$databasePath" ]]; then # Generate the new users file @@ -207,23 +200,12 @@ function syncElastic() { jq -r '.user + ":" + .data.hashed_password' \ >> "$usersTmpFile" [[ $? != 0 ]] && fail "Unable to read credential hashes from database" - - # Generate the new users_roles file - - echo "select 'superuser:' || ici.identifier " \ - "from identity_credential_identifiers ici, identity_credentials ic " \ - "where ici.identity_credential_id=ic.id and instr(ic.config, 'hashed_password') " \ - "order by ici.identifier;" | \ - sqlite3 "$databasePath" \ - >> "$rolesTmpFile" - [[ $? != 0 ]] && fail "Unable to read credential IDs from database" else echo "Database file does not exist yet, skipping users export" fi if [[ -s "${usersTmpFile}" ]]; then mv "${usersTmpFile}" "${elasticUsersFile}" - mv "${rolesTmpFile}" "${elasticRolesFile}" if [[ -z "$SKIP_STATE_APPLY" ]]; then echo "Elastic state will be re-applied to affected minions. This may take several minutes..." @@ -252,11 +234,73 @@ function listUsers() { response=$(curl -Ss -L ${kratosUrl}/identities) [[ $? != 0 ]] && fail "Unable to communicate with Kratos" - echo "${response}" | jq -r ".[] | .verifiable_addresses[0].value" | sort + users=$(echo "${response}" | jq -r ".[] | .verifiable_addresses[0].value" | sort) + for user in $users; do + roles=$(grep "$user" users_roles | cut -d: -f1 | tr '\n' ' ') + echo "$user: $roles" + done +} + +function addUserRole() { + email=$1 + role=$2 + + return adjustUserRole "$email" "$role" "add" +} + +function deleteUserRole() { + email=$1 + role=$2 + + return adjustUserRole "$email" "$role" "del" +} + +function adjustUserRole() { + email=$1 + role=$2 + op=$3 + + identityId=$(findIdByEmail "$email") + [[ ${identityId} == "" ]] && fail "User not found" + + if [ ! -f "$filename" ]; then + rolesTmpFile="${elasticRolesFile}.tmp" + createElasticFile "${rolesTmpFile}" + authPillarJson=$(lookup_salt_value "auth" "elasticsearch" "pillar" "json") + syncElasticSystemRole "$authPillarJson" "so_elastic_user" "superuser" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_kibana_user" "superuser" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_logstash_user" "superuser" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_beats_user" "superuser" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_collector" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_agent" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_monitor_user" "monitoring_user" "$rolesTmpFile" + mv "${rolesTmpFile}" "${elasticRolesFile}" + fi + + filename="$elasticRolesFile" + grep "$role:" "$elasticRolesFile" | grep "$email" && hasRole=1 + if [[ "$op" == "add" ]]; then + if [[ "$hasRole" -eq 1 ]]; then + fail "User '$email' already has the role: $role" + else + echo "$role:$email" >> "$filename" + fi + elif [[ "$op" == "del" ]]; then + if [[ "$hasRole" -ne 1 ]]; then + fail "User '$email' does not have the role: $role" + else + sed -i "/^$role:$email\$/d" "$filename" + fi + else + echo "Unsupported role adjustment operation: $op" + exit 1 + fi + return 0 } function createUser() { email=$1 + role=$1 now=$(date -u +%FT%TZ) addUserJson=$(cat < /opt/so/conf/elasticsearch/users && chown 930:930 /opt/so/conf/elasticsearch/users && chmod 600 /opt/so/conf/elasticsearch/users + - name: cat /opt/so/conf/elasticsearch/users.tmp > /opt/so/conf/elasticsearch/users && chown 930:939 /opt/so/conf/elasticsearch/users && chmod 660 /opt/so/conf/elasticsearch/users - onchanges: - file: /opt/so/conf/elasticsearch/users.tmp @@ -201,7 +218,7 @@ auth_users_roles_inode: require: - file: auth_users_roles cmd.run: - - name: cat /opt/so/conf/elasticsearch/users_roles.tmp > /opt/so/conf/elasticsearch/users_roles && chown 930:930 /opt/so/conf/elasticsearch/users_roles && chmod 600 /opt/so/conf/elasticsearch/users_roles + - name: cat /opt/so/conf/elasticsearch/users_roles.tmp > /opt/so/conf/elasticsearch/users_roles && chown 930:939 /opt/so/conf/elasticsearch/users_roles && chmod 660 /opt/so/conf/elasticsearch/users_roles - onchanges: - file: /opt/so/conf/elasticsearch/users_roles.tmp @@ -283,7 +300,7 @@ so-elasticsearch-pipelines: - file: esyml - file: so-elasticsearch-pipelines-file -{% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-node', 'so-import'] and TEMPLATES %} +{% if TEMPLATES %} so-elasticsearch-templates: cmd.run: - name: /usr/sbin/so-elasticsearch-templates-load @@ -291,6 +308,12 @@ so-elasticsearch-templates: - template: jinja {% endif %} +so-elasticsearch-roles-load: + cmd.run: + - name: /usr/sbin/so-elasticsearch-roles-load + - cwd: /opt/so + - template: jinja + {% endif %} {# if grains['role'] != 'so-helix' #} {% else %} diff --git a/salt/elasticsearch/roles/analyst.json b/salt/elasticsearch/roles/analyst.json new file mode 100644 index 000000000..f81c5a67e --- /dev/null +++ b/salt/elasticsearch/roles/analyst.json @@ -0,0 +1,45 @@ +{ + "elasticsearch": { + "cluster": [ + "cancel_task", + "create_snapshot", + "monitor", + "monitor_data_frame_transforms", + "monitor_ml", + "monitor_rollup", + "monitor_snapshot", + "monitor_text_structure", + "monitor_transform", + "monitor_watcher", + "read_ccr", + "read_ilm", + "read_pipeline", + "read_slm" + ], + "indices": [ + { + "names": [ + "so-*" + ], + "privileges": [ + "read", + "read_cross_cluster", + "monitor", + "view_index_metadata" + ] + } + ], + "run_as": [] + }, + "kibana": [ + { + "spaces": [ + "*" + ], + "base": [ + "read" + ], + "feature": {} + } + ] +} \ No newline at end of file diff --git a/salt/soc/files/kratos/schema.json b/salt/soc/files/kratos/schema.json index 986086936..19ee2197c 100644 --- a/salt/soc/files/kratos/schema.json +++ b/salt/soc/files/kratos/schema.json @@ -31,10 +31,6 @@ "type": "string", "title": "Last Name" }, - "role": { - "type": "string", - "title": "Role" - }, "status": { "type": "string", "title": "Status" diff --git a/salt/soc/files/soc/custom_roles b/salt/soc/files/soc/custom_roles new file mode 100644 index 000000000..80ae7b147 --- /dev/null +++ b/salt/soc/files/soc/custom_roles @@ -0,0 +1,20 @@ +# Define custom business role mappings, or remove mappings that come with +# the default SOC deployment. +# +# IMPORTANT: This file should be copied from the salt/default tree into +# the salt/local tree (preserving the same directory structure). +# Failure to do this will result in the customizations being +# overwritten on future upgrades. +# +# Syntax => prebuiltRoleX: customRoleY: op +# Explanation => roleY and roleZ are adjusted permissions of roleX, op is: +# + add the new permissions/role mappings (default) +# - remove existing prebuilt permissions +# +# In the example below, we will define a new role for junior analysts, +# that is nearly identical to the analyst role that comes with SOC, with the +# exception that it removes their ability to obtain details about other +# analysts in the system. +# +# analyst: jr_analyst +# user-monitor: jr_analyst:- diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index fc6d5f28d..6119f0e6b 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -85,6 +85,14 @@ "statickeyauth": { "anonymousCidr": "{{ DNET }}/24", "apiKey": "{{ SENSORONIKEY }}" + }, + "staticrbac": { + "roleFiles": [ + "rbac/permissions", + "rbac/roles", + "rbac/users_roles", + "rbac/custom_roles" + ] } }, "client": { diff --git a/salt/soc/init.sls b/salt/soc/init.sls index b8cdb09ba..c3c466849 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -62,6 +62,15 @@ soccustom: - mode: 600 - template: jinja +soccustomroles: + file.managed: + - name: /opt/so/conf/soc/custom_roles + - source: salt://soc/files/soc/custom_roles + - user: 939 + - group: 939 + - mode: 600 + - template: jinja + # we dont want this added too early in setup, so we add the onlyif to verify 'startup_states: highstate' # is in the minion config. That line is added before the final highstate during setup sosyncusers: @@ -81,6 +90,8 @@ so-soc: - /opt/so/conf/soc/motd.md:/opt/sensoroni/html/motd.md:ro - /opt/so/conf/soc/banner.md:/opt/sensoroni/html/login/banner.md:ro - /opt/so/conf/soc/custom.js:/opt/sensoroni/html/js/custom.js:ro + - /opt/so/conf/soc/custom_roles:/opt/sensoroni/rbac/custom_roles:ro + - /opt/so/conf/elasticsearch/users_roles:/opt/sensoroni/rbac/users_roles:ro - /opt/so/log/soc/:/opt/sensoroni/logs/:rw {%- if salt['pillar.get']('nodestab', {}) %} - extra_hosts: diff --git a/setup/so-functions b/setup/so-functions index 9a64a561e..d851c80b1 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -121,7 +121,7 @@ add_web_user() { { echo "Attempting to add administrator user for web interface..."; export SKIP_STATE_APPLY=true - echo "$WEBPASSWD1" | /usr/sbin/so-user add "$WEBUSER"; + echo "$WEBPASSWD1" | /usr/sbin/so-user add "$WEBUSER" "superuser"; unset SKIP_STATE_APPLY echo "Add user result: $?"; } >> "/root/so-user-add.log" 2>&1 From 56894b95813a4566ca18ab035882c513961ad5e4 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 2 Sep 2021 11:15:32 -0400 Subject: [PATCH 38/61] Add logic to check unattended flag when checking if updates are available --- salt/common/tools/sbin/soup | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index de26e73ea..17339fdd5 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -289,25 +289,31 @@ check_os_updates() { OSUPDATES=$(yum -q list updates | wc -l) fi if [[ "$OSUPDATES" -gt 0 ]]; then - echo $NEEDUPDATES - echo "" - read -p "Press U to update OS packages (recommended), C to continue without updates, or E to exit: " confirm - - if [[ "$confirm" == [cC] ]]; then + if [[ -z $UNATTENDED ]]; then + echo "$NEEDUPDATES" + echo "" + read -rp "Press U to update OS packages (recommended), C to continue without updates, or E to exit: " confirm + if [[ "$confirm" == [cC] ]]; then echo "Continuing without updating packages" - elif [[ "$confirm" == [uU] ]]; then + elif [[ "$confirm" == [uU] ]]; then echo "Applying Grid Updates" - set +e - run_check_net_err "salt '*' -b 5 state.apply patch.os queue=True" 'Could not apply OS updates, please check your network connection.' - set -e - else + update_flag=true + else echo "Exiting soup" exit 0 + fi + else + update_flag=true fi else - echo "Looks like you have an updated OS" + echo "Looks like you have an updated OS" + fi + + if [[ $update_flag == true ]]; then + set +e + run_check_net_err "salt '*' -b 5 state.apply patch.os queue=True" 'Could not apply OS updates, please check your network connection.' + set -e fi - } clean_dockers() { @@ -787,7 +793,7 @@ main() { echo "" check_airgap if [[ $is_airgap -eq 0 && $UNATTENDED == true && -z $ISOLOC ]]; then - echo "Missing file argument for unattended airgap upgrade." + echo "Missing file argument (-f ) for unattended airgap upgrade." exit 0 fi echo "Checking to see if this is a manager." From 2bf471054bb3c6aa0afde1c430ac7933c21427ce Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 2 Sep 2021 11:46:18 -0400 Subject: [PATCH 39/61] Cloudtrail Event Fields --- salt/soc/files/soc/hunt.eventfields.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/hunt.eventfields.json b/salt/soc/files/soc/hunt.eventfields.json index 354aa31ce..fa5f41ded 100644 --- a/salt/soc/files/soc/hunt.eventfields.json +++ b/salt/soc/files/soc/hunt.eventfields.json @@ -44,5 +44,6 @@ ":elasticsearch:": ["soc_timestamp", "agent.name", "message", "log.level", "metadata.version", "metadata.pipeline", "event.dataset" ], ":kibana:": ["soc_timestamp", "host.name", "message", "kibana.log.meta.req.headers.x-real-ip", "event.dataset" ], "::rootcheck": ["soc_timestamp", "host.name", "metadata.ip_address", "log.full", "event.dataset", "event.module" ], - "::syscollector": ["soc_timestamp", "host.name", "metadata.ip_address", "wazuh.data.type", "event.dataset", "event.module" ] + "::syscollector": ["soc_timestamp", "host.name", "metadata.ip_address", "wazuh.data.type", "event.dataset", "event.module" ], + ":aws:": ["soc_timestamp", "aws.cloudtrail.event_category", "aws.cloudtrail.event_type", "event.provider", "event.action", "event.outcome", "cloud.region", "user.name", "s ource.ip", "source.geo.region_iso_code" ] } From 1cee603ee4f56d4c3f1917f41122dbd79589a2f2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 2 Sep 2021 13:24:04 -0400 Subject: [PATCH 40/61] Squid event fields --- salt/soc/files/soc/hunt.eventfields.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/hunt.eventfields.json b/salt/soc/files/soc/hunt.eventfields.json index fa5f41ded..9eac130e8 100644 --- a/salt/soc/files/soc/hunt.eventfields.json +++ b/salt/soc/files/soc/hunt.eventfields.json @@ -45,5 +45,6 @@ ":kibana:": ["soc_timestamp", "host.name", "message", "kibana.log.meta.req.headers.x-real-ip", "event.dataset" ], "::rootcheck": ["soc_timestamp", "host.name", "metadata.ip_address", "log.full", "event.dataset", "event.module" ], "::syscollector": ["soc_timestamp", "host.name", "metadata.ip_address", "wazuh.data.type", "event.dataset", "event.module" ], - ":aws:": ["soc_timestamp", "aws.cloudtrail.event_category", "aws.cloudtrail.event_type", "event.provider", "event.action", "event.outcome", "cloud.region", "user.name", "s ource.ip", "source.geo.region_iso_code" ] + ":aws:": ["soc_timestamp", "aws.cloudtrail.event_category", "aws.cloudtrail.event_type", "event.provider", "event.action", "event.outcome", "cloud.region", "user.name", "source.ip", "source.geo.region_iso_code" ], + ":squid:": ["soc_timestamp", "url.original", "destination.ip", "destination.geo.country_iso_code", "user.name", "source.ip" ] } From 9f5dafd560c47b00b6bc71e2cc6bd3078d59975f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 2 Sep 2021 13:48:18 -0400 Subject: [PATCH 41/61] More Event Fields --- salt/soc/files/soc/hunt.eventfields.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/hunt.eventfields.json b/salt/soc/files/soc/hunt.eventfields.json index 9eac130e8..e519e78b6 100644 --- a/salt/soc/files/soc/hunt.eventfields.json +++ b/salt/soc/files/soc/hunt.eventfields.json @@ -30,7 +30,7 @@ "::software": ["soc_timestamp", "source.ip", "software.name", "software.type" ], "::ssh": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "ssh.version", "ssh.hassh_version", "ssh.direction", "ssh.client", "ssh.server", "log.id.uid" ], "::ssl": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "ssl.server_name", "ssl.certificate.subject", "ssl.validation_status", "ssl.version", "log.id.uid" ], - "::syslog": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "syslog.facility", "network.protocol", "syslog.severity", "log.id.uid" ], + ":zeek:syslog": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "syslog.facility", "network.protocol", "syslog.severity", "log.id.uid" ], "::tunnels": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "tunnel_type", "action", "log.id.uid" ], "::weird": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "weird.name", "log.id.uid" ], "::x509": ["soc_timestamp", "x509.certificate.subject", "x509.certificate.key.type", "x509.certificate.key.length", "x509.certificate.issuer", "log.id.fuid" ], @@ -44,7 +44,9 @@ ":elasticsearch:": ["soc_timestamp", "agent.name", "message", "log.level", "metadata.version", "metadata.pipeline", "event.dataset" ], ":kibana:": ["soc_timestamp", "host.name", "message", "kibana.log.meta.req.headers.x-real-ip", "event.dataset" ], "::rootcheck": ["soc_timestamp", "host.name", "metadata.ip_address", "log.full", "event.dataset", "event.module" ], + "::ossec": ["soc_timestamp", "host.name", "metadata.ip_address", "log.full", "event.dataset", "event.module" ], "::syscollector": ["soc_timestamp", "host.name", "metadata.ip_address", "wazuh.data.type", "event.dataset", "event.module" ], + ":syslog:syslog": ["soc_timestamp", "host.name", "metadata.ip_address", "real_message", "syslog.priority", "syslog.application" ], ":aws:": ["soc_timestamp", "aws.cloudtrail.event_category", "aws.cloudtrail.event_type", "event.provider", "event.action", "event.outcome", "cloud.region", "user.name", "source.ip", "source.geo.region_iso_code" ], ":squid:": ["soc_timestamp", "url.original", "destination.ip", "destination.geo.country_iso_code", "user.name", "source.ip" ] } From c4d402d8b429a8da407b8557a2e567b45bdac1c1 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 2 Sep 2021 15:45:47 -0400 Subject: [PATCH 42/61] Ensure role file exists before ES state is run --- salt/common/tools/sbin/so-user | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 7ec094efb..d60b04567 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -150,6 +150,23 @@ function createElasticFile() { chown "${esUID}:${esGID}" "$filename" } +function ensureRoleFileExists() { + if [ ! -f "$elasticRolesFile" ]; then + echo "Creating new roles file: $elasticRolesFile" + rolesTmpFile="${elasticRolesFile}.tmp" + createElasticFile "${rolesTmpFile}" + authPillarJson=$(lookup_salt_value "auth" "elasticsearch" "pillar" "json") + syncElasticSystemRole "$authPillarJson" "so_elastic_user" "superuser" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_kibana_user" "superuser" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_logstash_user" "superuser" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_beats_user" "superuser" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_collector" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_agent" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_monitor_user" "monitoring_user" "$rolesTmpFile" + mv "${rolesTmpFile}" "${elasticRolesFile}" + fi +} + function syncElasticSystemUser() { json=$1 userid=$2 @@ -179,6 +196,8 @@ function syncElasticSystemRole() { function syncElastic() { echo "Syncing users between SOC and Elastic..." + ensureRoleFileExists + usersTmpFile="${elasticUsersFile}.tmp" createElasticFile "${usersTmpFile}" @@ -263,19 +282,7 @@ function adjustUserRole() { identityId=$(findIdByEmail "$email") [[ ${identityId} == "" ]] && fail "User not found" - if [ ! -f "$filename" ]; then - rolesTmpFile="${elasticRolesFile}.tmp" - createElasticFile "${rolesTmpFile}" - authPillarJson=$(lookup_salt_value "auth" "elasticsearch" "pillar" "json") - syncElasticSystemRole "$authPillarJson" "so_elastic_user" "superuser" "$rolesTmpFile" - syncElasticSystemRole "$authPillarJson" "so_kibana_user" "superuser" "$rolesTmpFile" - syncElasticSystemRole "$authPillarJson" "so_logstash_user" "superuser" "$rolesTmpFile" - syncElasticSystemRole "$authPillarJson" "so_beats_user" "superuser" "$rolesTmpFile" - syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_collector" "$rolesTmpFile" - syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_agent" "$rolesTmpFile" - syncElasticSystemRole "$authPillarJson" "so_monitor_user" "monitoring_user" "$rolesTmpFile" - mv "${rolesTmpFile}" "${elasticRolesFile}" - fi + ensureRoleFileExists filename="$elasticRolesFile" grep "$role:" "$elasticRolesFile" | grep "$email" && hasRole=1 From ce70380f0ff4d19130709b96e7d7b4bd4edf39f8 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 2 Sep 2021 17:59:33 -0400 Subject: [PATCH 43/61] resolve so-user errors from recent auth changes --- salt/common/tools/sbin/so-user | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index d60b04567..d7f9c2e6b 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -20,7 +20,7 @@ source $(dirname $0)/so-common DEFAULT_ROLE=analyst -if [[ $# -lt 1 || $# -gt 2 ]]; then +if [[ $# -lt 1 || $# -gt 3 ]]; then echo "Usage: $0 [email] [role]" echo "" echo " where is one of the following:" @@ -197,7 +197,7 @@ function syncElasticSystemRole() { function syncElastic() { echo "Syncing users between SOC and Elastic..." ensureRoleFileExists - + usersTmpFile="${elasticUsersFile}.tmp" createElasticFile "${usersTmpFile}" @@ -255,7 +255,7 @@ function listUsers() { users=$(echo "${response}" | jq -r ".[] | .verifiable_addresses[0].value" | sort) for user in $users; do - roles=$(grep "$user" users_roles | cut -d: -f1 | tr '\n' ' ') + roles=$(grep "$user" "$elasticRolesFile" | cut -d: -f1 | tr '\n' ' ') echo "$user: $roles" done } @@ -264,14 +264,14 @@ function addUserRole() { email=$1 role=$2 - return adjustUserRole "$email" "$role" "add" + adjustUserRole "$email" "$role" "add" } function deleteUserRole() { email=$1 role=$2 - return adjustUserRole "$email" "$role" "del" + adjustUserRole "$email" "$role" "del" } function adjustUserRole() { @@ -285,9 +285,10 @@ function adjustUserRole() { ensureRoleFileExists filename="$elasticRolesFile" - grep "$role:" "$elasticRolesFile" | grep "$email" && hasRole=1 + hasRole=0 + grep "$role:" "$elasticRolesFile" | grep -q "$email" && hasRole=1 if [[ "$op" == "add" ]]; then - if [[ "$hasRole" -eq 1 ]]; then + if [[ "$hasRole" == "1" ]]; then fail "User '$email' already has the role: $role" else echo "$role:$email" >> "$filename" @@ -299,15 +300,13 @@ function adjustUserRole() { sed -i "/^$role:$email\$/d" "$filename" fi else - echo "Unsupported role adjustment operation: $op" - exit 1 + fail "Unsupported role adjustment operation: $op" fi - return 0 } function createUser() { email=$1 - role=$1 + role=$2 now=$(date -u +%FT%TZ) addUserJson=$(cat < Date: Thu, 2 Sep 2021 19:12:32 -0400 Subject: [PATCH 44/61] Consolidate password validation messaging --- salt/common/tools/sbin/so-common | 9 +++++++++ salt/common/tools/sbin/so-fleet-user-add | 5 +---- salt/common/tools/sbin/so-thehive-user-add | 5 +---- salt/common/tools/sbin/so-user | 2 +- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 7ad74ad49..fe97c9b27 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -99,6 +99,15 @@ check_password() { return $? } +check_password_and_exit() { + local password=$1 + if ! check_password "$password"; then + echo "Password is invalid. Do not include single quotes, double quotes, dollar signs, and backslashes in the password." + exit 2 + fi + return 0 +} + check_elastic_license() { [ -n "$TESTING" ] && return diff --git a/salt/common/tools/sbin/so-fleet-user-add b/salt/common/tools/sbin/so-fleet-user-add index 9d80c2076..8ce7325c3 100755 --- a/salt/common/tools/sbin/so-fleet-user-add +++ b/salt/common/tools/sbin/so-fleet-user-add @@ -41,10 +41,7 @@ if [[ $? == 0 ]]; then fi read -rs FLEET_PASS -if ! check_password "$FLEET_PASS"; then - echo "Password is invalid. Please exclude single quotes, double quotes, dollar signs, and backslashes from the password." - exit 2 -fi +check_password_and_exit "$FLEET_PASS" FLEET_HASH=$(docker exec so-soctopus python -c "import bcrypt; print(bcrypt.hashpw('$FLEET_PASS'.encode('utf-8'), bcrypt.gensalt()).decode('utf-8'));" 2>&1) if [[ $? -ne 0 ]]; then diff --git a/salt/common/tools/sbin/so-thehive-user-add b/salt/common/tools/sbin/so-thehive-user-add index e26dc58fc..9cbe0cd56 100755 --- a/salt/common/tools/sbin/so-thehive-user-add +++ b/salt/common/tools/sbin/so-thehive-user-add @@ -41,10 +41,7 @@ if [[ $? == 0 ]]; then fi read -rs THEHIVE_PASS -if ! check_password "$THEHIVE_PASS"; then - echo "Password is invalid. Please exclude single quotes, double quotes, dollar signs, and backslashes from the password." - exit 2 -fi +check_password_and_exit "$THEHIVE_PASS" # Create new user in TheHive resp=$(curl -sk -XPOST -H "Authorization: Bearer $THEHIVE_KEY" -H "Content-Type: application/json" -L "https://$THEHVIE_API_URL/user" -d "{\"login\" : \"$THEHIVE_USER\",\"name\" : \"$THEHIVE_USER\",\"roles\" : [\"read\",\"alert\",\"write\",\"admin\"],\"preferences\" : \"{}\",\"password\" : \"$THEHIVE_PASS\"}") diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index d7f9c2e6b..757ca10c1 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -98,7 +98,7 @@ function validatePassword() { if [[ $len -lt 6 ]]; then echo "Password does not meet the minimum requirements" exit 2 - fi + check_password_and_exit "$password" } function validateEmail() { From 649f339934628fed1452f84f6c727140bdc4d991 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 2 Sep 2021 20:30:48 -0400 Subject: [PATCH 45/61] Correct typo --- salt/common/tools/sbin/so-user | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 757ca10c1..b3d65b128 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -98,6 +98,7 @@ function validatePassword() { if [[ $len -lt 6 ]]; then echo "Password does not meet the minimum requirements" exit 2 + fi check_password_and_exit "$password" } From 9fb28709d5b9b8f57ae36c776727ae32d8784870 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 3 Sep 2021 10:47:00 -0400 Subject: [PATCH 46/61] Add maxfiles to the steno config --- salt/pcap/files/config | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/pcap/files/config b/salt/pcap/files/config index 048775ef7..900234bc1 100644 --- a/salt/pcap/files/config +++ b/salt/pcap/files/config @@ -1,11 +1,13 @@ {%- set interface = salt['pillar.get']('sensor:interface', 'bond0') %} {%- set diskfreepercentage = salt['pillar.get']('steno:diskfreepercentage', 10) %} +{%- set maxfiles = salt['pillar.get']('steno:maxfiles', 30000) %} + { "Threads": [ { "PacketsDirectory": "/nsm/pcap" , "IndexDirectory": "/nsm/pcapindex" - , "MaxDirectoryFiles": 30000 + , "MaxDirectoryFiles": {{ maxfiles }} , "DiskFreePercentage": {{ diskfreepercentage }} } ] @@ -15,4 +17,4 @@ , "Host": "127.0.0.1" , "Flags": ["-v", "--uid=stenographer", "--gid=stenographer"{{ BPF_COMPILED }}] , "CertPath": "/etc/stenographer/certs" -} \ No newline at end of file +} From fbbb7f4e85fdc31427e83483b4062c8ee41cfa4d Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 3 Sep 2021 15:54:05 -0400 Subject: [PATCH 47/61] Add auditor role; update analyst role with correct syntax --- .../tools/sbin/so-elasticsearch-roles-load | 2 +- salt/elasticsearch/roles/analyst.json | 97 +++++++++++-------- 2 files changed, 58 insertions(+), 41 deletions(-) diff --git a/salt/common/tools/sbin/so-elasticsearch-roles-load b/salt/common/tools/sbin/so-elasticsearch-roles-load index 3b0f580fe..d094163ee 100644 --- a/salt/common/tools/sbin/so-elasticsearch-roles-load +++ b/salt/common/tools/sbin/so-elasticsearch-roles-load @@ -51,7 +51,7 @@ cd ${ELASTICSEARCH_ROLES} echo "Loading templates..." for role in *; do name=$(echo "$role" | cut -d. -f1) - so-elasticsearch-query security/roles/$name -XPUT -d @"$role" + so-elasticsearch-query _security/role/$name -XPUT -d @"$role" done cd - >/dev/null diff --git a/salt/elasticsearch/roles/analyst.json b/salt/elasticsearch/roles/analyst.json index f81c5a67e..3fbaa9489 100644 --- a/salt/elasticsearch/roles/analyst.json +++ b/salt/elasticsearch/roles/analyst.json @@ -1,45 +1,62 @@ { - "elasticsearch": { - "cluster": [ - "cancel_task", - "create_snapshot", - "monitor", - "monitor_data_frame_transforms", - "monitor_ml", - "monitor_rollup", - "monitor_snapshot", - "monitor_text_structure", - "monitor_transform", - "monitor_watcher", - "read_ccr", - "read_ilm", - "read_pipeline", - "read_slm" - ], - "indices": [ - { - "names": [ - "so-*" - ], - "privileges": [ - "read", - "read_cross_cluster", - "monitor", - "view_index_metadata" - ] - } - ], - "run_as": [] - }, - "kibana": [ + "cluster": [ + "cancel_task", + "create_snapshot", + "monitor", + "monitor_data_frame_transforms", + "monitor_ml", + "monitor_rollup", + "monitor_snapshot", + "monitor_text_structure", + "monitor_transform", + "monitor_watcher", + "read_ccr", + "read_ilm", + "read_pipeline", + "read_slm" + ], + "indices": [ { - "spaces": [ - "*" + "names": [ + "so-*" ], - "base": [ - "read" - ], - "feature": {} + "privileges": [ + "index", + "read", + "read_cross_cluster", + "monitor", + "view_index_metadata" + ] } - ] + ], + "applications": [ + { + "application": "kibana-.kibana", + "privileges": [ + "feature_discover.all", + "feature_dashboard.all", + "feature_canvas.all", + "feature_maps.all", + "feature_ml.all", + "feature_logs.read", + "feature_visualize.all", + "feature_infrastructure.read", + "feature_apm.read", + "feature_uptime.read", + "feature_siem.read", + "feature_dev_tools.read", + "feature_advancedSettings.read", + "feature_indexPatterns.read", + "feature_savedObjectsManagement.read", + "feature_savedObjectsTagging.read", + "feature_fleet.all", + "feature_actions.read", + "feature_stackAlerts.read" + ], + "resources": [ + "*" + ] + } + ], + "run_as": [] } \ No newline at end of file From 94ea1f856b4ce8cb1db30321a7a866de2ab70475 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 3 Sep 2021 15:59:48 -0400 Subject: [PATCH 48/61] Add auditor role; update analyst role with correct syntax --- salt/elasticsearch/roles/auditor.json | 59 +++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 salt/elasticsearch/roles/auditor.json diff --git a/salt/elasticsearch/roles/auditor.json b/salt/elasticsearch/roles/auditor.json new file mode 100644 index 000000000..26df1207c --- /dev/null +++ b/salt/elasticsearch/roles/auditor.json @@ -0,0 +1,59 @@ +{ + "cluster": [ + "monitor", + "monitor_data_frame_transforms", + "monitor_ml", + "monitor_rollup", + "monitor_snapshot", + "monitor_text_structure", + "monitor_transform", + "monitor_watcher", + "read_ccr", + "read_ilm", + "read_pipeline", + "read_slm" + ], + "indices": [ + { + "names": [ + "so-*" + ], + "privileges": [ + "read", + "read_cross_cluster", + "monitor", + "view_index_metadata" + ] + } + ], + "applications": [ + { + "application": "kibana-.kibana", + "privileges": [ + "feature_discover.read", + "feature_dashboard.read", + "feature_canvas.read", + "feature_maps.read", + "feature_ml.read", + "feature_logs.read", + "feature_visualize.read", + "feature_infrastructure.read", + "feature_apm.read", + "feature_uptime.read", + "feature_siem.read", + "feature_dev_tools.read", + "feature_advancedSettings.read", + "feature_indexPatterns.read", + "feature_savedObjectsManagement.read", + "feature_savedObjectsTagging.read", + "feature_fleet.read", + "feature_actions.read", + "feature_stackAlerts.read" + ], + "resources": [ + "*" + ] + } + ], + "run_as": [] +} \ No newline at end of file From b7591093cfbc0ba74ee95a6dbcc3705860496f9f Mon Sep 17 00:00:00 2001 From: Rob Waight <43173714+rwaight@users.noreply.github.com> Date: Sat, 4 Sep 2021 09:45:03 -0400 Subject: [PATCH 49/61] Add index sorting to so-common-template.json Add index sorting to so-common-template.json --- salt/elasticsearch/templates/so/so-common-template.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index 777bf3f53..c9772b5db 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -8,6 +8,8 @@ "index.refresh_interval":"30s", "index.routing.allocation.require.box_type":"hot", "index.mapping.total_fields.limit": "1500", + "index.sort.field": "@timestamp", + "index.sort.order": "desc", "analysis": { "analyzer": { "es_security_analyzer": { From e3900606dc3c3fe3a83892f448cb96dab75d35ab Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Sat, 4 Sep 2021 10:42:18 -0400 Subject: [PATCH 50/61] Enable index sorting by default but allow it to be disabled --- salt/elasticsearch/templates/so/so-common-template.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index c9772b5db..4394ebb65 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -1,3 +1,4 @@ +{%- set INDEX_SORTING = salt['pillar.get']('elasticsearch:index_sorting', True) %} { "index_patterns": ["so-*"], "version":50001, @@ -8,8 +9,10 @@ "index.refresh_interval":"30s", "index.routing.allocation.require.box_type":"hot", "index.mapping.total_fields.limit": "1500", +{%- if INDEX_SORTING is sameas true %} "index.sort.field": "@timestamp", "index.sort.order": "desc", +{%- endif %} "analysis": { "analyzer": { "es_security_analyzer": { From 3c59579f99ae23398d3d1bf5f12a03e127de7469 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 7 Sep 2021 13:03:30 -0400 Subject: [PATCH 51/61] Add maintenance privilege for analysts to refresh indices --- salt/elasticsearch/roles/analyst.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/roles/analyst.json b/salt/elasticsearch/roles/analyst.json index 3fbaa9489..fc788e2f1 100644 --- a/salt/elasticsearch/roles/analyst.json +++ b/salt/elasticsearch/roles/analyst.json @@ -22,9 +22,10 @@ ], "privileges": [ "index", + "maintenance", + "monitor", "read", "read_cross_cluster", - "monitor", "view_index_metadata" ] } From b8600be0f102c351c6260b44748ad69c79da6b17 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 8 Sep 2021 12:12:09 -0400 Subject: [PATCH 52/61] Incude server.publicBaseUrl --- salt/kibana/etc/kibana.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/kibana/etc/kibana.yml b/salt/kibana/etc/kibana.yml index 6fcafe68f..2819f301d 100644 --- a/salt/kibana/etc/kibana.yml +++ b/salt/kibana/etc/kibana.yml @@ -3,9 +3,11 @@ {%- set ES = salt['pillar.get']('manager:mainip', '') -%} {%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- set URLBASE = salt['pillar.get']('global:url_base') %} server.name: kibana server.host: "0" server.basePath: /kibana +server.publicBaseUrl: https://{{ URLBASE }}/kibana elasticsearch.hosts: [ "https://{{ ES }}:9200" ] elasticsearch.ssl.verificationMode: none #kibana.index: ".kibana" From 5d235e932c02a2f3b06771c42c09e73cf5fc080b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 9 Sep 2021 10:46:28 -0400 Subject: [PATCH 53/61] Fix Raid Status for cloud --- salt/common/tools/sbin/so-raid-status | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/common/tools/sbin/so-raid-status b/salt/common/tools/sbin/so-raid-status index e817554d3..54c2b3024 100755 --- a/salt/common/tools/sbin/so-raid-status +++ b/salt/common/tools/sbin/so-raid-status @@ -20,6 +20,10 @@ appliance_check() { {%- if salt['grains.get']('sosmodel', '') %} APPLIANCE=1 + {%- if grains['sosmodel'] in ['SO2AMI01', 'SO2GCI01', 'SO2AZI01'] %} + echo "nsmraid=0" > /opt/so/log/raid/status.log + exit 0 + {%- endif %} DUDEYOUGOTADELL=$(dmidecode |grep Dell) if [[ -n $DUDEYOUGOTADELL ]]; then APPTYPE=dell From c83f119cc08124dc41859ade5e01e00ba08e4bf6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 9 Sep 2021 10:59:35 -0400 Subject: [PATCH 54/61] Update so-raid-status --- salt/common/tools/sbin/so-raid-status | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/common/tools/sbin/so-raid-status b/salt/common/tools/sbin/so-raid-status index 54c2b3024..4664a5961 100755 --- a/salt/common/tools/sbin/so-raid-status +++ b/salt/common/tools/sbin/so-raid-status @@ -21,7 +21,6 @@ appliance_check() { {%- if salt['grains.get']('sosmodel', '') %} APPLIANCE=1 {%- if grains['sosmodel'] in ['SO2AMI01', 'SO2GCI01', 'SO2AZI01'] %} - echo "nsmraid=0" > /opt/so/log/raid/status.log exit 0 {%- endif %} DUDEYOUGOTADELL=$(dmidecode |grep Dell) From 63f67b35009204606a5b30894d9629f89912cd61 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 9 Sep 2021 14:16:05 -0400 Subject: [PATCH 55/61] Rephrase screen that warns about more RAM requirements --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 125335729..c7954e65f 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -114,7 +114,7 @@ whiptail_components_adv_warning() { [ -n "$TESTING" ] && return - whiptail --title "$whiptail_title" --msgbox "Please keep in mind the more services that you enable the more RAM that is required." 8 75 + whiptail --title "$whiptail_title" --msgbox "The next screen will provide the ability to choose optional services to be enabled for this installation. Be aware that the more services you enable the more RAM that is required." 8 75 } whiptail_create_admin_user() { From 94688a9adb4ba7fa2ddee20eb0c5d9b915658a30 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 9 Sep 2021 14:29:09 -0400 Subject: [PATCH 56/61] Eliminate adv component popup --- setup/so-setup | 1 - setup/so-whiptail | 12 +++--------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 958d8aea1..7d5ae0a0e 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -473,7 +473,6 @@ if [[ $is_manager && ! $is_eval ]]; then fi if [[ $is_manager ]]; then - whiptail_components_adv_warning whiptail_enable_components if [[ "$STRELKA" = 1 ]]; then diff --git a/setup/so-whiptail b/setup/so-whiptail index c7954e65f..1f0986119 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -110,13 +110,6 @@ whiptail_check_exitstatus() { esac } -whiptail_components_adv_warning() { - - [ -n "$TESTING" ] && return - - whiptail --title "$whiptail_title" --msgbox "The next screen will provide the ability to choose optional services to be enabled for this installation. Be aware that the more services you enable the more RAM that is required." 8 75 -} - whiptail_create_admin_user() { [ -n "$TESTING" ] && return @@ -408,9 +401,10 @@ whiptail_enable_components() { PLAYBOOK=0 STRELKA=0 +description="Choose optional services to be enabled for this installation. Be aware that the more services you enable the more RAM that is required." if [[ $is_eval ]]; then COMPONENTS=$(whiptail --title "$whiptail_title" --checklist \ - "Select Components to install:" 20 75 8 \ + "$description" 20 75 8 \ GRAFANA "Enable Grafana for system monitoring" ON \ OSQUERY "Enable Fleet with osquery" ON \ WAZUH "Enable Wazuh" ON \ @@ -419,7 +413,7 @@ if [[ $is_eval ]]; then STRELKA "Enable Strelka" ON 3>&1 1>&2 2>&3) else COMPONENTS=$(whiptail --title "$whiptail_title" --checklist \ - "Select Components to install:" 20 75 7 \ + "$description" 20 75 7 \ OSQUERY "Enable Fleet with osquery" ON \ WAZUH "Enable Wazuh" ON \ THEHIVE "Enable TheHive" ON \ From db50ef71b4297605b4d1eb7daef2b41c0d4c5cb6 Mon Sep 17 00:00:00 2001 From: doug Date: Fri, 10 Sep 2021 06:19:16 -0400 Subject: [PATCH 57/61] FEATURE: Add docs link to Setup #5459 --- setup/so-whiptail | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 1f0986119..ed4067da1 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1867,8 +1867,9 @@ whiptail_you_sure() { read -r -d '' you_sure_text <<- EOM Welcome to Security Onion Setup! - You can use Setup for lots of different use cases from a small standalone installation to a large distributed deployment for your enterprise. - + You can use Setup for lots of different use cases from a small standalone installation to a large distributed deployment for your enterprise. Don't forget to review the documentation at: + https://docs.securityonion.net + Setup uses keyboard navigation and you can use arrow keys to move around. Certain screens may provide a list and ask you to select one or more items from that list. You can use [SPACE] to select items and [ENTER] to proceed to the next screen. Would you like to continue? From 4f6a3269cb0e5c8c1378cf46c7b84bc389722b45 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 10 Sep 2021 09:59:47 -0400 Subject: [PATCH 58/61] Add more detail to syscollector --- salt/soc/files/soc/hunt.eventfields.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/files/soc/hunt.eventfields.json b/salt/soc/files/soc/hunt.eventfields.json index e519e78b6..7964a360b 100644 --- a/salt/soc/files/soc/hunt.eventfields.json +++ b/salt/soc/files/soc/hunt.eventfields.json @@ -45,7 +45,7 @@ ":kibana:": ["soc_timestamp", "host.name", "message", "kibana.log.meta.req.headers.x-real-ip", "event.dataset" ], "::rootcheck": ["soc_timestamp", "host.name", "metadata.ip_address", "log.full", "event.dataset", "event.module" ], "::ossec": ["soc_timestamp", "host.name", "metadata.ip_address", "log.full", "event.dataset", "event.module" ], - "::syscollector": ["soc_timestamp", "host.name", "metadata.ip_address", "wazuh.data.type", "event.dataset", "event.module" ], + "::syscollector": ["soc_timestamp", "host.name", "metadata.ip_address", "wazuh.data.type", "log.full", "event.dataset", "event.module" ], ":syslog:syslog": ["soc_timestamp", "host.name", "metadata.ip_address", "real_message", "syslog.priority", "syslog.application" ], ":aws:": ["soc_timestamp", "aws.cloudtrail.event_category", "aws.cloudtrail.event_type", "event.provider", "event.action", "event.outcome", "cloud.region", "user.name", "source.ip", "source.geo.region_iso_code" ], ":squid:": ["soc_timestamp", "url.original", "destination.ip", "destination.geo.country_iso_code", "user.name", "source.ip" ] From 9c310de459eeafab4913a8dd0533a53b0a5c5a17 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 10 Sep 2021 13:15:09 -0400 Subject: [PATCH 59/61] Add `--force` flag to idstools-rulecat under so-rule-update This forces idstools to pull from the url each time, which prevents it from clearing all.rules if idstools-rulecat is run twice within 15 minutes by any method (either restarting the container or running so-rule-update) --- salt/common/tools/sbin/so-rule-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-rule-update b/salt/common/tools/sbin/so-rule-update index 397719d61..a4e4d3cd7 100755 --- a/salt/common/tools/sbin/so-rule-update +++ b/salt/common/tools/sbin/so-rule-update @@ -10,4 +10,4 @@ got_root() { } got_root -docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat $1" +docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat --force $1" From 0b42b19763093f3597a6905138571a0a5777d6c7 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 10 Sep 2021 13:41:58 -0400 Subject: [PATCH 60/61] Update so-rule-update to source so-common --- salt/common/tools/sbin/so-rule-update | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/salt/common/tools/sbin/so-rule-update b/salt/common/tools/sbin/so-rule-update index a4e4d3cd7..0fc691d36 100755 --- a/salt/common/tools/sbin/so-rule-update +++ b/salt/common/tools/sbin/so-rule-update @@ -1,13 +1,7 @@ #!/bin/bash -got_root() { - # Make sure you are root - if [ "$(id -u)" -ne 0 ]; then - echo "This script must be run using sudo!" - exit 1 - fi +. /usr/sbin/so-common -} got_root docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat --force $1" From 679faddd52469395a2b0ece8ab64c24cd67584ba Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 10 Sep 2021 13:44:37 -0400 Subject: [PATCH 61/61] Update so-rule-update to pass all args to docker exec Instead of passing $1, build a string from all args and add that to the command string for the docker exec statement --- salt/common/tools/sbin/so-rule-update | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-rule-update b/salt/common/tools/sbin/so-rule-update index 0fc691d36..a3c2616a4 100755 --- a/salt/common/tools/sbin/so-rule-update +++ b/salt/common/tools/sbin/so-rule-update @@ -2,6 +2,9 @@ . /usr/sbin/so-common +argstr="" +for arg in "$@"; do + argstr="${argstr} \"${arg}\"" +done -got_root -docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat --force $1" +docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat --force ${argstr}"