From 6f9a263af316bce698feaa227910a489988c63ca Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 20 Aug 2021 10:40:15 -0400 Subject: [PATCH 01/51] remove the curator cronjobs if it is disabled --- salt/curator/init.sls | 89 ++++++++++++++++++++++++++++--------------- 1 file changed, 59 insertions(+), 30 deletions(-) diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 9f0dc44f4..1acea5d25 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -4,6 +4,7 @@ {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} +{% set REMOVECURATORCRON = False %} {% if grains['role'] in ['so-eval', 'so-node', 'so-managersearch', 'so-heavynode', 'so-standalone', 'so-manager'] %} {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} {% from "curator/map.jinja" import CURATOROPTIONS with context %} @@ -88,36 +89,6 @@ curdel: - group: 939 - mode: 755 -so-curatorcloseddeletecron: - cron.present: - - name: /usr/sbin/so-curator-closed-delete > /opt/so/log/curator/cron-closed-delete.log 2>&1 - - user: root - - minute: '*' - - hour: '*' - - daymonth: '*' - - month: '*' - - dayweek: '*' - -so-curatorclosecron: - cron.present: - - name: /usr/sbin/so-curator-close > /opt/so/log/curator/cron-close.log 2>&1 - - user: root - - minute: '*' - - hour: '*' - - daymonth: '*' - - month: '*' - - dayweek: '*' - -so-curatordeletecron: - cron.present: - - name: /usr/sbin/so-curator-delete > /opt/so/log/curator/cron-delete.log 2>&1 - - user: root - - minute: '*' - - hour: '*' - - daymonth: '*' - - month: '*' - - dayweek: '*' - so-curator: docker_container.{{ CURATOROPTIONS.status }}: {% if CURATOROPTIONS.status == 'running' %} @@ -152,11 +123,15 @@ so-curator_so-status.disabled: file.comment: - name: /opt/so/conf/so-status/so-status.conf - regex: ^so-curator$ + + # need to remove cronjobs here since curator is disabled + {% set REMOVECURATORCRON = True %} {% else %} delete_so-curator_so-status.disabled: file.uncomment: - name: /opt/so/conf/so-status/so-status.conf - regex: ^so-curator$ + {% endif %} {% else %} @@ -165,6 +140,60 @@ delete_so-curator_so-status: - name: /opt/so/conf/so-status/so-status.conf - match: ^so-curator$ - mode: delete + + # need to remove cronjobs here since curator is disabled + {% set REMOVECURATORCRON = True %} + + {% endif %} + + {% if REMOVECURATORCRON %} +so-curatorcloseddeletecron: + cron.absent: + - name: /usr/sbin/so-curator-closed-delete > /opt/so/log/curator/cron-closed-delete.log 2>&1 + - user: root + +so-curatorclosecron: + cron.absent: + - name: /usr/sbin/so-curator-close > /opt/so/log/curator/cron-close.log 2>&1 + - user: root + +so-curatordeletecron: + cron.absent: + - name: /usr/sbin/so-curator-delete > /opt/so/log/curator/cron-delete.log 2>&1 + - user: root + + {% else %} + +so-curatorcloseddeletecron: + cron.present: + - name: /usr/sbin/so-curator-closed-delete > /opt/so/log/curator/cron-closed-delete.log 2>&1 + - user: root + - minute: '*' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' + +so-curatorclosecron: + cron.present: + - name: /usr/sbin/so-curator-close > /opt/so/log/curator/cron-close.log 2>&1 + - user: root + - minute: '*' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' + +so-curatordeletecron: + cron.present: + - name: /usr/sbin/so-curator-delete > /opt/so/log/curator/cron-delete.log 2>&1 + - user: root + - minute: '*' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' + {% endif %} # Begin Curator Cron Jobs From 936c998ecbcb22b778b7f836d0d19e14f18ac207 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 20 Aug 2021 12:49:55 -0400 Subject: [PATCH 02/51] CURATOR ISO info --- VERIFY_ISO.md | 22 +++++++++++----------- sigs/securityonion-2.3.70-CURATOR.iso.sig | Bin 0 -> 543 bytes 2 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 sigs/securityonion-2.3.70-CURATOR.iso.sig diff --git a/VERIFY_ISO.md b/VERIFY_ISO.md index 16dbd024f..0f78a7a39 100644 --- a/VERIFY_ISO.md +++ b/VERIFY_ISO.md @@ -1,18 +1,18 @@ -### 2.3.70 ISO image built on 2021/08/17 +### 2.3.70-CURATOR ISO image built on 2021/08/20 ### Download and Verify -2.3.70 ISO image: -https://download.securityonion.net/file/securityonion/securityonion-2.3.70.iso +2.3.70-CURATOR ISO image: +https://download.securityonion.net/file/securityonion/securityonion-2.3.70-CURATOR.iso -MD5: F048FABC7FD2D0E1A8B02381F115D1E0 -SHA1: DF6D20FEF13CDC1B19309D2A1178D6E5D25FDA6F -SHA256: B193FFD7EE69958A8E257117149DCFB2125C5772FBFA6003AD80FD1CC129E571 +MD5: FECB0156F1E3BC9BD1E074FF8C6B2B6D +SHA1: 38C1AF89C2CFFD7777E4E7A7C8DA1FEC5BB163D5 +SHA256: 38835C6096E859E3FDD1253678312AEC15BD95A08F681B86D84990A4094B48DC Signature for ISO image: -https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.70.iso.sig +https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.70-CURATOR.iso.sig Signing key: https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS @@ -26,22 +26,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/ma Download the signature file for the ISO: ``` -wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.70.iso.sig +wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.70-CURATOR.iso.sig ``` Download the ISO image: ``` -wget https://download.securityonion.net/file/securityonion/securityonion-2.3.70.iso +wget https://download.securityonion.net/file/securityonion/securityonion-2.3.70-CURATOR.iso ``` Verify the downloaded ISO image using the signature file: ``` -gpg --verify securityonion-2.3.70.iso.sig securityonion-2.3.70.iso +gpg --verify securityonion-2.3.70-CURATOR.iso.sig securityonion-2.3.70-CURATOR.iso ``` The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Tue 17 Aug 2021 10:52:17 PM EDT using RSA key ID FE507013 +gpg: Signature made Fri 20 Aug 2021 11:26:28 AM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/sigs/securityonion-2.3.70-CURATOR.iso.sig b/sigs/securityonion-2.3.70-CURATOR.iso.sig new file mode 100644 index 0000000000000000000000000000000000000000..1e13fa860f3e5e08201a5fa006fce12df379dc55 GIT binary patch literal 543 zcmV+)0^t3L0vrSY0RjL91p;9o$)o@Z2@re`V7LBIa1#^P5B@eCH@S!5mo*3@4p;WB z1d0&VgKYoxcOQPqT0K@Lud+(=N(QU~ciT$V(eu|!7JYOC_;|Lh+!9+v#}d()5hi6) zX)9QSE+~u@N6{;7xMn}Ljlpou=@V0tquh} zT7gllVaG_(2-aEJEHT2{4d;)OE(CF6t4Ot3=8C}plx=yAI+P}8k+;uw6M>Mu4Kq(m zP?~0=CzQ}jA~b@A8^gkQzoG&ZZkd~{Ct>5j2(%Xya}!oq9>q##(2Vv*#g$r13C~zm zJmyV3yy?qT$pBwCryZ;AdrBW>#J8ML>oQEaeGVgYv7`o(QRh(Ptm46%JAr@B*`^{B z5(L`97{4a>F3{l>G52Adj0x{zE#%>+s$dPMc_4*|R_lyVA{eP&rNdVSR?u>&!Q;Ht zI=2x%fuIjbaQC5ow!Lq;xEAoG!9d!f>5Izt1)?!i`?$jh8;A%K%t2m0Z@yL{Kss|y zE=?R&>x}7AK4E!|Hc<~2V5?_<&&%iN_Wl3> literal 0 HcmV?d00001 From 51b517581a9be262992d68a8650f2da85cc12858 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 20 Aug 2021 15:38:56 -0400 Subject: [PATCH 03/51] 2.3.70 sigs --- VERIFY_ISO.md | 8 ++++---- sigs/securityonion-2.3.70-CURATOR.iso.sig | Bin 543 -> 543 bytes 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERIFY_ISO.md b/VERIFY_ISO.md index 0f78a7a39..7b75e8583 100644 --- a/VERIFY_ISO.md +++ b/VERIFY_ISO.md @@ -7,9 +7,9 @@ 2.3.70-CURATOR ISO image: https://download.securityonion.net/file/securityonion/securityonion-2.3.70-CURATOR.iso -MD5: FECB0156F1E3BC9BD1E074FF8C6B2B6D -SHA1: 38C1AF89C2CFFD7777E4E7A7C8DA1FEC5BB163D5 -SHA256: 38835C6096E859E3FDD1253678312AEC15BD95A08F681B86D84990A4094B48DC +MD5: E0F7882E37B1B6BC4F9A6C8FD6F213F6 +SHA1: 82E1204BAD9489B275A083A642F175E352F9A332 +SHA256: 147CA7F5082273EDCC32EF6322D86A04CCB2E96B3A7F0B01EFA8A029BD84C3D7 Signature for ISO image: https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.70-CURATOR.iso.sig @@ -41,7 +41,7 @@ gpg --verify securityonion-2.3.70-CURATOR.iso.sig securityonion-2.3.70-CURATOR.i The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Fri 20 Aug 2021 11:26:28 AM EDT using RSA key ID FE507013 +gpg: Signature made Fri 20 Aug 2021 01:23:59 PM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/sigs/securityonion-2.3.70-CURATOR.iso.sig b/sigs/securityonion-2.3.70-CURATOR.iso.sig index 1e13fa860f3e5e08201a5fa006fce12df379dc55..a9dfc3d1d7f0f0d0cdb701e34a1bed95a9942479 100644 GIT binary patch literal 543 zcmV+)0^t3L0vrSY0RjL91p;9o|+^87z{GoYD2B2{aO?q5hc{#Nr* zxgk!wiJq;?2rQC>ZWv_>rE8Y5;D%ShhA(q+@@%KZBk`Jj=i2Uz4jkgQGM9SU_R;Qh zX|I^fwgdW3$v?xvX|zlK!{ZRh!0iJh!#fjc@yoNN7mqdE zvp6=~$M&s`H7N;Y3mUp>s0Nf->)%Z5X-6bbBxj>frxQ1ORjwr<g*aqZ0$~ViB#!zf~3!g6ovVG67Ym00=UL(3;W%+mm7<5aj)s5*Lvz;(>vFS9f#J zfeUUN8$#?3^eGjuCDz_bi^qV$X*1g5P{1U_nSH%1qJ{B literal 543 zcmV+)0^t3L0vrSY0RjL91p;9o$)o@Z2@re`V7LBIa1#^P5B@eCH@S!5mo*3@4p;WB z1d0&VgKYoxcOQPqT0K@Lud+(=N(QU~ciT$V(eu|!7JYOC_;|Lh+!9+v#}d()5hi6) zX)9QSE+~u@N6{;7xMn}Ljlpou=@V0tquh} zT7gllVaG_(2-aEJEHT2{4d;)OE(CF6t4Ot3=8C}plx=yAI+P}8k+;uw6M>Mu4Kq(m zP?~0=CzQ}jA~b@A8^gkQzoG&ZZkd~{Ct>5j2(%Xya}!oq9>q##(2Vv*#g$r13C~zm zJmyV3yy?qT$pBwCryZ;AdrBW>#J8ML>oQEaeGVgYv7`o(QRh(Ptm46%JAr@B*`^{B z5(L`97{4a>F3{l>G52Adj0x{zE#%>+s$dPMc_4*|R_lyVA{eP&rNdVSR?u>&!Q;Ht zI=2x%fuIjbaQC5ow!Lq;xEAoG!9d!f>5Izt1)?!i`?$jh8;A%K%t2m0Z@yL{Kss|y zE=?R&>x}7AK4E!|Hc<~2V5?_<&&%iN_Wl3> From aa420b914b2bcc42f2d1d520a7b8b187e01dcc8f Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 20 Aug 2021 16:27:09 -0400 Subject: [PATCH 04/51] Initial version so-playbook-import --- salt/common/tools/sbin/so-playbook-import | 22 ++++++++++++++++++++++ salt/soctopus/init.sls | 3 ++- 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 salt/common/tools/sbin/so-playbook-import diff --git a/salt/common/tools/sbin/so-playbook-import b/salt/common/tools/sbin/so-playbook-import new file mode 100644 index 000000000..107851278 --- /dev/null +++ b/salt/common/tools/sbin/so-playbook-import @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +ENABLEPLAY=${1:-False} + +docker exec so-soctopus /usr/local/bin/python -c "import playbook; print(playbook.play_import($ENABLEPLAY))" diff --git a/salt/soctopus/init.sls b/salt/soctopus/init.sls index c2c8dc1ac..724e5a617 100644 --- a/salt/soctopus/init.sls +++ b/salt/soctopus/init.sls @@ -10,7 +10,7 @@ soctopusdir: file.directory: - - name: /opt/so/conf/soctopus + - name: /opt/so/conf/soctopus/sigma-import - user: 939 - group: 939 - makedirs: True @@ -63,6 +63,7 @@ so-soctopus: - /opt/so/log/soctopus/:/var/log/SOCtopus/:rw - /opt/so/rules/elastalert/playbook:/etc/playbook-rules:rw - /opt/so/conf/navigator/nav_layer_playbook.json:/etc/playbook/nav_layer_playbook.json:rw + - /opt/so/conf/soctopus/sigma-import/:/SOCtopus/sigma-import/:rw {% if ISAIRGAP is sameas true %} - /nsm/repo/rules/sigma:/soctopus/sigma {% endif %} From b78da5c2377c4e1d282f74a01d47bbd43b35d5c0 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 23 Aug 2021 09:36:20 -0400 Subject: [PATCH 05/51] Merge hotfix to dev; reset to .80 --- HOTFIX | 2 -- VERSION | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/HOTFIX b/HOTFIX index 393a7e913..e69de29bb 100644 --- a/HOTFIX +++ b/HOTFIX @@ -1,2 +0,0 @@ - -CURATOR diff --git a/VERSION b/VERSION index e183d6a6c..d5c0d128d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3.70 +2.3.80 From 19cfce5e0bc83e41a1adbbaa313876e34df03d5e Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 23 Aug 2021 10:47:41 -0400 Subject: [PATCH 06/51] Add curator delete yml files --- salt/curator/files/action/so-beats-delete.yml | 29 +++++++++++++++++++ .../files/action/so-firewall-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-ids-delete.yml | 29 +++++++++++++++++++ .../curator/files/action/so-import-delete.yml | 29 +++++++++++++++++++ .../files/action/so-osquery-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-ossec-delete.yml | 29 +++++++++++++++++++ .../files/action/so-strelka-delete.yml | 29 +++++++++++++++++++ .../curator/files/action/so-syslog-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-zeek-delete.yml | 29 +++++++++++++++++++ 9 files changed, 261 insertions(+) create mode 100644 salt/curator/files/action/so-beats-delete.yml create mode 100644 salt/curator/files/action/so-firewall-delete.yml create mode 100644 salt/curator/files/action/so-ids-delete.yml create mode 100644 salt/curator/files/action/so-import-delete.yml create mode 100644 salt/curator/files/action/so-osquery-delete.yml create mode 100644 salt/curator/files/action/so-ossec-delete.yml create mode 100644 salt/curator/files/action/so-strelka-delete.yml create mode 100644 salt/curator/files/action/so-syslog-delete.yml create mode 100644 salt/curator/files/action/so-zeek-delete.yml diff --git a/salt/curator/files/action/so-beats-delete.yml b/salt/curator/files/action/so-beats-delete.yml new file mode 100644 index 000000000..77931d661 --- /dev/null +++ b/salt/curator/files/action/so-beats-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-beats:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete beats indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-beats.*|so-beats.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-firewall-delete.yml b/salt/curator/files/action/so-firewall-delete.yml new file mode 100644 index 000000000..7588de437 --- /dev/null +++ b/salt/curator/files/action/so-firewall-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-firewall:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete firewall indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-firewall.*|so-firewall.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-ids-delete.yml b/salt/curator/files/action/so-ids-delete.yml new file mode 100644 index 000000000..e5bda4e34 --- /dev/null +++ b/salt/curator/files/action/so-ids-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-ids:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete IDS indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-ids.*|so-ids.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-import-delete.yml b/salt/curator/files/action/so-import-delete.yml new file mode 100644 index 000000000..aa9808c5f --- /dev/null +++ b/salt/curator/files/action/so-import-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-import:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete import indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-import.*|so-import.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-osquery-delete.yml b/salt/curator/files/action/so-osquery-delete.yml new file mode 100644 index 000000000..d77b1b3d1 --- /dev/null +++ b/salt/curator/files/action/so-osquery-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-osquery:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete import indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-osquery.*|so-osquery.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-ossec-delete.yml b/salt/curator/files/action/so-ossec-delete.yml new file mode 100644 index 000000000..7aea13e41 --- /dev/null +++ b/salt/curator/files/action/so-ossec-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-ossec:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete ossec indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-ossec.*|so-ossec.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-strelka-delete.yml b/salt/curator/files/action/so-strelka-delete.yml new file mode 100644 index 000000000..3487aeb6d --- /dev/null +++ b/salt/curator/files/action/so-strelka-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-strelka:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete Strelka indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-strelka.*|so-strelka.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-syslog-delete.yml b/salt/curator/files/action/so-syslog-delete.yml new file mode 100644 index 000000000..5fe7417ad --- /dev/null +++ b/salt/curator/files/action/so-syslog-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-syslog:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete syslog indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-syslog.*|so-syslog.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-zeek-delete.yml b/salt/curator/files/action/so-zeek-delete.yml new file mode 100644 index 000000000..0694c2aed --- /dev/null +++ b/salt/curator/files/action/so-zeek-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete Zeek indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-zeek.*|so-zeek.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file From b053f29a89aabf5cb98e621820cffc1184ce0c48 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 23 Aug 2021 12:58:52 -0400 Subject: [PATCH 07/51] only create dashboards for certain node types - https://github.com/Security-Onion-Solutions/securityonion/issues/5268 --- salt/grafana/init.sls | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/grafana/init.sls b/salt/grafana/init.sls index b74fa2495..eca1786cc 100644 --- a/salt/grafana/init.sls +++ b/salt/grafana/init.sls @@ -12,16 +12,21 @@ {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] or (grains.role == 'so-eval' and GRAFANA == 1) %} +{% set ALLOWED_DASHBOARDS = ['overview', 'standalone', 'manager', 'managersearch', 'sensor', 'searchnode', 'heavynode', 'eval'] %} {% set DASHBOARDS = ['overview'] %} {% if grains.role == 'so-eval' %} {% do DASHBOARDS.append('eval') %} {% else %} {# Grab a unique listing of nodetypes that exists so that we create only the needed dashboards #} {% for dashboard in salt['cmd.shell']("ls /opt/so/saltstack/local/pillar/minions/|awk -F'_' {'print $2'}|awk -F'.' {'print $1'}").split() %} - {% do DASHBOARDS.append(dashboard) %} + {% if dashboard id ALLOWED_DASHBOARDS %} + {% do DASHBOARDS.append(dashboard) %} + {% endif %} {% endfor %} {% endif %} + + # Grafana all the things grafanadir: file.directory: From a9cc68f89e82d9db7263f449188c5b07b4c37890 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 23 Aug 2021 13:02:49 -0400 Subject: [PATCH 08/51] add unique identifier for hotfix - https://github.com/Security-Onion-Solutions/securityonion/issues/5268 --- HOTFIX | 1 + 1 file changed, 1 insertion(+) diff --git a/HOTFIX b/HOTFIX index 393a7e913..7e7c96536 100644 --- a/HOTFIX +++ b/HOTFIX @@ -1,2 +1,3 @@ CURATOR +GRAFANA_DASH_ALLOW From 80f4d0325487c033dcb587aeffae03b4b5711601 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 23 Aug 2021 13:05:28 -0400 Subject: [PATCH 09/51] place unique identifier on same line for hotfix - https://github.com/Security-Onion-Solutions/securityonion/issues/5268 --- HOTFIX | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/HOTFIX b/HOTFIX index 7e7c96536..100732042 100644 --- a/HOTFIX +++ b/HOTFIX @@ -1,3 +1,2 @@ -CURATOR -GRAFANA_DASH_ALLOW +CURATOR GRAFANA_DASH_ALLOW From f87cf123b065ba67365ca49cca679569e1abd9eb Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 23 Aug 2021 13:08:11 -0400 Subject: [PATCH 10/51] fix typo - https://github.com/Security-Onion-Solutions/securityonion/issues/5268 --- salt/grafana/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/grafana/init.sls b/salt/grafana/init.sls index eca1786cc..42df29af4 100644 --- a/salt/grafana/init.sls +++ b/salt/grafana/init.sls @@ -19,7 +19,7 @@ {% else %} {# Grab a unique listing of nodetypes that exists so that we create only the needed dashboards #} {% for dashboard in salt['cmd.shell']("ls /opt/so/saltstack/local/pillar/minions/|awk -F'_' {'print $2'}|awk -F'.' {'print $1'}").split() %} - {% if dashboard id ALLOWED_DASHBOARDS %} + {% if dashboard in ALLOWED_DASHBOARDS %} {% do DASHBOARDS.append(dashboard) %} {% endif %} {% endfor %} From 6d49bca0ac4640a4251cc360625bd3a25bcd3f92 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 23 Aug 2021 15:36:11 -0400 Subject: [PATCH 11/51] Switch to new auth params --- salt/curator/files/curator.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/curator/files/curator.yml b/salt/curator/files/curator.yml index 15ecb8cb1..5ec4bdc5f 100644 --- a/salt/curator/files/curator.yml +++ b/salt/curator/files/curator.yml @@ -19,7 +19,8 @@ client: - {{elasticsearch}} port: 9200 {%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} - http_auth: {{ ES_USER }}:{{ ES_PASS }} + username: {{ ES_USER }} + password: {{ ES_PASS }} {%- endif %} url_prefix: use_ssl: True From 2f8b68e67ad22cf5576d1da0a244b995ce02b691 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 24 Aug 2021 09:58:28 -0400 Subject: [PATCH 12/51] sig files and iso info --- VERIFY_ISO.md | 22 +++++++++++----------- sigs/securityonion-2.3.70-GRAFANA.iso.sig | Bin 0 -> 543 bytes 2 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 sigs/securityonion-2.3.70-GRAFANA.iso.sig diff --git a/VERIFY_ISO.md b/VERIFY_ISO.md index 7b75e8583..ccece2d2d 100644 --- a/VERIFY_ISO.md +++ b/VERIFY_ISO.md @@ -1,18 +1,18 @@ -### 2.3.70-CURATOR ISO image built on 2021/08/20 +### 2.3.70-GRAFANA ISO image built on 2021/08/20 ### Download and Verify -2.3.70-CURATOR ISO image: -https://download.securityonion.net/file/securityonion/securityonion-2.3.70-CURATOR.iso +2.3.70-GRAFANA ISO image: +https://download.securityonion.net/file/securityonion/securityonion-2.3.70-GRAFANA.iso -MD5: E0F7882E37B1B6BC4F9A6C8FD6F213F6 -SHA1: 82E1204BAD9489B275A083A642F175E352F9A332 -SHA256: 147CA7F5082273EDCC32EF6322D86A04CCB2E96B3A7F0B01EFA8A029BD84C3D7 +MD5: A16683FC8F2151C290E359FC6066B1F2 +SHA1: A93329C103CCCE665968F246163FBE5D41EF0510 +SHA256: 3ED0177CADF203324363916AA240A10C58DC3E9044A9ADE173A80674701A50A3 Signature for ISO image: -https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.70-CURATOR.iso.sig +https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.70-GRAFANA.iso.sig Signing key: https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS @@ -26,22 +26,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/ma Download the signature file for the ISO: ``` -wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.70-CURATOR.iso.sig +wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.70-GRAFANA.iso.sig ``` Download the ISO image: ``` -wget https://download.securityonion.net/file/securityonion/securityonion-2.3.70-CURATOR.iso +wget https://download.securityonion.net/file/securityonion/securityonion-2.3.70-GRAFANA.iso ``` Verify the downloaded ISO image using the signature file: ``` -gpg --verify securityonion-2.3.70-CURATOR.iso.sig securityonion-2.3.70-CURATOR.iso +gpg --verify securityonion-2.3.70-GRAFANA.iso.sig securityonion-2.3.70-GRAFANA.iso ``` The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Fri 20 Aug 2021 01:23:59 PM EDT using RSA key ID FE507013 +gpg: Signature made Mon 23 Aug 2021 01:43:00 PM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/sigs/securityonion-2.3.70-GRAFANA.iso.sig b/sigs/securityonion-2.3.70-GRAFANA.iso.sig new file mode 100644 index 0000000000000000000000000000000000000000..8abec2097636d1a0a617713e6242fc3da0601c32 GIT binary patch literal 543 zcmV+)0^t3L0vrSY0RjL91p;9s-Xs7D2@re`V7LBIa1&*n5C44arQs$-!yk#kQT4^& zU9S!CIy8L7Tmw<*3Q_bFL(v!z&N?J8Nbi6QN7|&bad=ps=Rs9BE7B+iq)6tF)Nsbz z1V!;S77Pj6#DJ_jfX1SU&^>!1JO_NZD&>WAQzo^jhD-XmEJC-L@>O~Y4fDABv|SNf~gLi7`LIE)9Gh2-8l3k^e|g7{DGq^79)&VbTg-#pc>sd$4k}aszK3 zBYelljMlBthc$cXW9d0PAk7~~_7%MXBRJq zBzV6kK_(&RVgr8cYB!x@wDj>gX&L(izTn9v3V@01yPqWH98?vp;*&K!s=qn&Nsk{* z%8%LKgo62LH!dgf(fy literal 0 HcmV?d00001 From 6e921415eaeff7f9b30afc676aec3e956bf9c899 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 24 Aug 2021 10:00:06 -0400 Subject: [PATCH 13/51] sig files and iso info --- VERIFY_ISO.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERIFY_ISO.md b/VERIFY_ISO.md index ccece2d2d..2be975430 100644 --- a/VERIFY_ISO.md +++ b/VERIFY_ISO.md @@ -1,4 +1,4 @@ -### 2.3.70-GRAFANA ISO image built on 2021/08/20 +### 2.3.70-GRAFANA ISO image built on 2021/08/23 From 9783d13ea3556b530a7cdb31ead3165d6175485d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 24 Aug 2021 10:22:01 -0400 Subject: [PATCH 14/51] remove identifier from HOTFIX file --- HOTFIX | 2 -- 1 file changed, 2 deletions(-) diff --git a/HOTFIX b/HOTFIX index 100732042..e69de29bb 100644 --- a/HOTFIX +++ b/HOTFIX @@ -1,2 +0,0 @@ - -CURATOR GRAFANA_DASH_ALLOW From de5f823abff6ac67e4fc6f50cede4ca44da08c84 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 25 Aug 2021 18:28:17 -0400 Subject: [PATCH 15/51] Add automation for deploy-vader env --- .../automation/vader-distributed-iso-manager | 76 ++++++++++++++++++ setup/automation/vader-distributed-iso-search | 78 ++++++++++++++++++ setup/automation/vader-distributed-iso-sensor | 80 +++++++++++++++++++ 3 files changed, 234 insertions(+) create mode 100644 setup/automation/vader-distributed-iso-manager create mode 100644 setup/automation/vader-distributed-iso-search create mode 100644 setup/automation/vader-distributed-iso-sensor diff --git a/setup/automation/vader-distributed-iso-manager b/setup/automation/vader-distributed-iso-manager new file mode 100644 index 000000000..07a22b588 --- /dev/null +++ b/setup/automation/vader-distributed-iso-manager @@ -0,0 +1,76 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +TESTING=true + +address_type=DHCP +ADMINUSER=onionuser +ADMINPASS1=onionuser +ADMINPASS2=onionuser +ALLOW_CIDR=0.0.0.0/0 +ALLOW_ROLE=a +BASICZEEK=7 +BASICSURI=7 +# BLOGS= +#BNICS=eth1 +ZEEKVERSION=ZEEK +# CURCLOSEDAYS= +# EVALADVANCED=BASIC +GRAFANA=1 +# HELIXAPIKEY= +HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 +HNSENSOR=inherit +HOSTNAME=distributed-manager +install_type=MANAGER +# LSINPUTBATCHCOUNT= +# LSINPUTTHREADS= +# LSPIPELINEBATCH= +# LSPIPELINEWORKERS= +MANAGERADV=BASIC +# MDNS= +# MGATEWAY= +# MIP= +# MMASK= +MNIC=eth0 +# MSEARCH= +# MSRV= +# MTU= +NIDS=Suricata +# NODE_ES_HEAP_SIZE= +# NODE_LS_HEAP_SIZE= +NODESETUP=NODEBASIC +NSMSETUP=BASIC +NODEUPDATES=MANAGER +# OINKCODE= +OSQUERY=1 +# PATCHSCHEDULEDAYS= +# PATCHSCHEDULEHOURS= +PATCHSCHEDULENAME=auto +PLAYBOOK=1 +# REDIRECTHOST= +REDIRECTINFO=IP +RULESETUP=ETOPEN +# SHARDCOUNT= +# SKIP_REBOOT= +SOREMOTEPASS1=onionuser +SOREMOTEPASS2=onionuser +STRELKA=1 +THEHIVE=1 +WAZUH=1 +WEBUSER=onionuser@somewhere.invalid +WEBPASSWD1=0n10nus3r +WEBPASSWD2=0n10nus3r diff --git a/setup/automation/vader-distributed-iso-search b/setup/automation/vader-distributed-iso-search new file mode 100644 index 000000000..c1ced1537 --- /dev/null +++ b/setup/automation/vader-distributed-iso-search @@ -0,0 +1,78 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +TESTING=true + +address_type=DHCP +ADMINUSER=onionuser +ADMINPASS1=onionuser +ADMINPASS2=onionuser +# ALLOW_CIDR=0.0.0.0/0 +# ALLOW_ROLE=a +# BASICZEEK=7 +# BASICSURI=7 +# BLOGS= +# BNICS=eth1 +# ZEEKVERSION=ZEEK +# CURCLOSEDAYS= +# EVALADVANCED=BASIC +# GRAFANA=1 +# HELIXAPIKEY= +HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 +HNSENSOR=inherit +HOSTNAME=distributed-search +install_type=SEARCHNODE +# LSINPUTBATCHCOUNT= +# LSINPUTTHREADS= +# LSPIPELINEBATCH= +# LSPIPELINEWORKERS= +# MANAGERADV=BASIC +MANAGERUPDATES=1 +# MDNS= +# MGATEWAY= +# MIP= +# MMASK= +MNIC=eth0 +# MSEARCH= +MSRV=distributed-manager +MSRVIP=10.66.166.100 +# MTU= +# NIDS=Suricata +# NODE_ES_HEAP_SIZE= +# NODE_LS_HEAP_SIZE= +NODESETUP=NODEBASIC +NSMSETUP=ADVANCED +NODEUPDATES=MANAGER +# OINKCODE= +# OSQUERY=1 +# PATCHSCHEDULEDAYS= +# PATCHSCHEDULEHOURS= +PATCHSCHEDULENAME=auto +# PLAYBOOK=1 +# REDIRECTHOST= +# REDIRECTINFO=IP +# RULESETUP=ETOPEN +# SHARDCOUNT= +# SKIP_REBOOT= +SOREMOTEPASS1=onionuser +SOREMOTEPASS2=onionuser +# STRELKA=1 +# THEHIVE=1 +# WAZUH=1 +# WEBUSER=onionuser@somewhere.invalid +# WEBPASSWD1=0n10nus3r +# WEBPASSWD2=0n10nus3r diff --git a/setup/automation/vader-distributed-iso-sensor b/setup/automation/vader-distributed-iso-sensor new file mode 100644 index 000000000..356e67a2d --- /dev/null +++ b/setup/automation/vader-distributed-iso-sensor @@ -0,0 +1,80 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +TESTING=true + +address_type=DHCP +ADMINUSER=onionuser +ADMINPASS1=onionuser +ADMINPASS2=onionuser +# ALLOW_CIDR=0.0.0.0/0 +# ALLOW_ROLE=a +BASICZEEK=2 +BASICSURI=2 +# BLOGS= +BNICS=eth1 +ZEEKVERSION=ZEEK +# CURCLOSEDAYS= +# EVALADVANCED=BASIC +# GRAFANA=1 +# HELIXAPIKEY= +HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 +HNSENSOR=inherit +HOSTNAME=Distributed-Sensor +install_type=SENSOR +# LSINPUTBATCHCOUNT= +# LSINPUTTHREADS= +# LSPIPELINEBATCH= +# LSPIPELINEWORKERS= +# MANAGERADV=BASIC +MANAGERUPDATES=1 +# MDNS= +# MGATEWAY= +# MIP= +# MMASK= +MNIC=eth0 +# MSEARCH= +MSRV=distributed-manager +MSRVIP=10.66.166.100 +MTU=1500 +# NIDS=Suricata +# NODE_ES_HEAP_SIZE= +# NODE_LS_HEAP_SIZE= +# NODESETUP=NODEBASIC +NSMSETUP=ADVANCED +NODEUPDATES=MANAGER +# OINKCODE= +# OSQUERY=1 +# PATCHSCHEDULEDAYS= +# PATCHSCHEDULEHOURS= +PATCHSCHEDULENAME=auto +# PLAYBOOK=1 +# REDIRECTHOST= +# REDIRECTINFO=IP +# RULESETUP=ETOPEN +# SHARDCOUNT= +# SKIP_REBOOT= +SOREMOTEPASS1=onionuser +SOREMOTEPASS2=onionuser +# STRELKA=1 +SURIPINS=(2 3) +# THEHIVE=1 +# WAZUH=1 +# WEBUSER=onionuser@somewhere.invalid +# WEBPASSWD1=0n10nus3r +# WEBPASSWD2=0n10nus3r +ZEEKPINS=(0 1) From 09d50978379ab7ee6a8cf2080cf6532772edb09e Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 25 Aug 2021 21:08:49 -0400 Subject: [PATCH 16/51] Remove unused automation files --- .../automation/vader-distributed-iso-manager | 76 ------------------ setup/automation/vader-distributed-iso-search | 78 ------------------ setup/automation/vader-distributed-iso-sensor | 80 ------------------- 3 files changed, 234 deletions(-) delete mode 100644 setup/automation/vader-distributed-iso-manager delete mode 100644 setup/automation/vader-distributed-iso-search delete mode 100644 setup/automation/vader-distributed-iso-sensor diff --git a/setup/automation/vader-distributed-iso-manager b/setup/automation/vader-distributed-iso-manager deleted file mode 100644 index 07a22b588..000000000 --- a/setup/automation/vader-distributed-iso-manager +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash - -# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -TESTING=true - -address_type=DHCP -ADMINUSER=onionuser -ADMINPASS1=onionuser -ADMINPASS2=onionuser -ALLOW_CIDR=0.0.0.0/0 -ALLOW_ROLE=a -BASICZEEK=7 -BASICSURI=7 -# BLOGS= -#BNICS=eth1 -ZEEKVERSION=ZEEK -# CURCLOSEDAYS= -# EVALADVANCED=BASIC -GRAFANA=1 -# HELIXAPIKEY= -HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 -HNSENSOR=inherit -HOSTNAME=distributed-manager -install_type=MANAGER -# LSINPUTBATCHCOUNT= -# LSINPUTTHREADS= -# LSPIPELINEBATCH= -# LSPIPELINEWORKERS= -MANAGERADV=BASIC -# MDNS= -# MGATEWAY= -# MIP= -# MMASK= -MNIC=eth0 -# MSEARCH= -# MSRV= -# MTU= -NIDS=Suricata -# NODE_ES_HEAP_SIZE= -# NODE_LS_HEAP_SIZE= -NODESETUP=NODEBASIC -NSMSETUP=BASIC -NODEUPDATES=MANAGER -# OINKCODE= -OSQUERY=1 -# PATCHSCHEDULEDAYS= -# PATCHSCHEDULEHOURS= -PATCHSCHEDULENAME=auto -PLAYBOOK=1 -# REDIRECTHOST= -REDIRECTINFO=IP -RULESETUP=ETOPEN -# SHARDCOUNT= -# SKIP_REBOOT= -SOREMOTEPASS1=onionuser -SOREMOTEPASS2=onionuser -STRELKA=1 -THEHIVE=1 -WAZUH=1 -WEBUSER=onionuser@somewhere.invalid -WEBPASSWD1=0n10nus3r -WEBPASSWD2=0n10nus3r diff --git a/setup/automation/vader-distributed-iso-search b/setup/automation/vader-distributed-iso-search deleted file mode 100644 index c1ced1537..000000000 --- a/setup/automation/vader-distributed-iso-search +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash - -# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -TESTING=true - -address_type=DHCP -ADMINUSER=onionuser -ADMINPASS1=onionuser -ADMINPASS2=onionuser -# ALLOW_CIDR=0.0.0.0/0 -# ALLOW_ROLE=a -# BASICZEEK=7 -# BASICSURI=7 -# BLOGS= -# BNICS=eth1 -# ZEEKVERSION=ZEEK -# CURCLOSEDAYS= -# EVALADVANCED=BASIC -# GRAFANA=1 -# HELIXAPIKEY= -HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 -HNSENSOR=inherit -HOSTNAME=distributed-search -install_type=SEARCHNODE -# LSINPUTBATCHCOUNT= -# LSINPUTTHREADS= -# LSPIPELINEBATCH= -# LSPIPELINEWORKERS= -# MANAGERADV=BASIC -MANAGERUPDATES=1 -# MDNS= -# MGATEWAY= -# MIP= -# MMASK= -MNIC=eth0 -# MSEARCH= -MSRV=distributed-manager -MSRVIP=10.66.166.100 -# MTU= -# NIDS=Suricata -# NODE_ES_HEAP_SIZE= -# NODE_LS_HEAP_SIZE= -NODESETUP=NODEBASIC -NSMSETUP=ADVANCED -NODEUPDATES=MANAGER -# OINKCODE= -# OSQUERY=1 -# PATCHSCHEDULEDAYS= -# PATCHSCHEDULEHOURS= -PATCHSCHEDULENAME=auto -# PLAYBOOK=1 -# REDIRECTHOST= -# REDIRECTINFO=IP -# RULESETUP=ETOPEN -# SHARDCOUNT= -# SKIP_REBOOT= -SOREMOTEPASS1=onionuser -SOREMOTEPASS2=onionuser -# STRELKA=1 -# THEHIVE=1 -# WAZUH=1 -# WEBUSER=onionuser@somewhere.invalid -# WEBPASSWD1=0n10nus3r -# WEBPASSWD2=0n10nus3r diff --git a/setup/automation/vader-distributed-iso-sensor b/setup/automation/vader-distributed-iso-sensor deleted file mode 100644 index 356e67a2d..000000000 --- a/setup/automation/vader-distributed-iso-sensor +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash - -# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -TESTING=true - -address_type=DHCP -ADMINUSER=onionuser -ADMINPASS1=onionuser -ADMINPASS2=onionuser -# ALLOW_CIDR=0.0.0.0/0 -# ALLOW_ROLE=a -BASICZEEK=2 -BASICSURI=2 -# BLOGS= -BNICS=eth1 -ZEEKVERSION=ZEEK -# CURCLOSEDAYS= -# EVALADVANCED=BASIC -# GRAFANA=1 -# HELIXAPIKEY= -HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 -HNSENSOR=inherit -HOSTNAME=Distributed-Sensor -install_type=SENSOR -# LSINPUTBATCHCOUNT= -# LSINPUTTHREADS= -# LSPIPELINEBATCH= -# LSPIPELINEWORKERS= -# MANAGERADV=BASIC -MANAGERUPDATES=1 -# MDNS= -# MGATEWAY= -# MIP= -# MMASK= -MNIC=eth0 -# MSEARCH= -MSRV=distributed-manager -MSRVIP=10.66.166.100 -MTU=1500 -# NIDS=Suricata -# NODE_ES_HEAP_SIZE= -# NODE_LS_HEAP_SIZE= -# NODESETUP=NODEBASIC -NSMSETUP=ADVANCED -NODEUPDATES=MANAGER -# OINKCODE= -# OSQUERY=1 -# PATCHSCHEDULEDAYS= -# PATCHSCHEDULEHOURS= -PATCHSCHEDULENAME=auto -# PLAYBOOK=1 -# REDIRECTHOST= -# REDIRECTINFO=IP -# RULESETUP=ETOPEN -# SHARDCOUNT= -# SKIP_REBOOT= -SOREMOTEPASS1=onionuser -SOREMOTEPASS2=onionuser -# STRELKA=1 -SURIPINS=(2 3) -# THEHIVE=1 -# WAZUH=1 -# WEBUSER=onionuser@somewhere.invalid -# WEBPASSWD1=0n10nus3r -# WEBPASSWD2=0n10nus3r -ZEEKPINS=(0 1) From c5d120293df57e96af46195f33d9c4a4f3630c24 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 27 Aug 2021 11:33:51 -0400 Subject: [PATCH 17/51] Initial work to add unattended option to soup --- salt/common/tools/sbin/soup | 91 +++++++++++++++++++++++-------------- 1 file changed, 58 insertions(+), 33 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index ce8923e90..8adabf908 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -105,17 +105,20 @@ add_common() { airgap_mounted() { # Let's see if the ISO is already mounted. - if [ -f /tmp/soagupdate/SecurityOnion/VERSION ]; then + if [[ -f /tmp/soagupdate/SecurityOnion/VERSION ]]; then echo "The ISO is already mounted" else - echo "" - cat << EOF + if [[ -z $ISOLOC ]]; then + echo "This is airgap. Ask for a location." + echo "" + cat << EOF In order for soup to proceed, the path to the downloaded Security Onion ISO file, or the path to the CD-ROM or equivalent device containing the ISO media must be provided. For example, if you have copied the new Security Onion ISO file to your home directory, then the path might look like /home/myuser/securityonion-2.x.y.iso. Or, if you have burned the new ISO onto an optical disk then the path might look like /dev/cdrom. EOF - read -rp 'Enter the path to the new Security Onion ISO content: ' ISOLOC + read -rp 'Enter the path to the new Security Onion ISO content: ' ISOLOC + fi if [[ -f $ISOLOC ]]; then # Mounting the ISO image mkdir -p /tmp/soagupdate @@ -124,23 +127,27 @@ EOF if [ ! -f /tmp/soagupdate/SecurityOnion/VERSION ]; then echo "Something went wrong trying to mount the ISO." echo "Ensure you verify the ISO that you downloaded." - exit 0 + exit 1 else echo "ISO has been mounted!" fi elif [[ -f $ISOLOC/SecurityOnion/VERSION ]]; then ln -s $ISOLOC /tmp/soagupdate echo "Found the update content" - else + elif [[ -b $ISOLOC ]]; then mkdir -p /tmp/soagupdate mount $ISOLOC /tmp/soagupdate if [ ! -f /tmp/soagupdate/SecurityOnion/VERSION ]; then echo "Something went wrong trying to mount the device." echo "Ensure you verify the ISO that you downloaded." - exit 0 + exit 1 else echo "Device has been mounted!" - fi + fi + else + echo "Could not find Security Onion ISO content at ${ISOLOC}" + echo "Ensure the path you entered is correct, and that you verify the ISO that you downloaded." + exit 1 fi fi } @@ -774,39 +781,22 @@ verify_latest_update_script() { } main() { - set -e - set +e trap 'check_err $?' EXIT - echo "### Preparing soup at $(date) ###" - while getopts ":b" opt; do - case "$opt" in - b ) # process option b - shift - BATCHSIZE=$1 - if ! [[ "$BATCHSIZE" =~ ^[0-9]+$ ]]; then - echo "Batch size must be a number greater than 0." - exit 1 - fi - ;; - \? ) - echo "Usage: cmd [-b]" - ;; - esac - done - + echo "Checking to see if this is an airgap install." + echo "" + check_airgap + if [[ $is_airgap -eq 0 && $UNATTENDED == true && -z $ISOLOC ]]; then + echo "Missing file argument for unattended airgap upgrade." + fi echo "Checking to see if this is a manager." echo "" require_manager set_minionid - echo "Checking to see if this is an airgap install." - echo "" - check_airgap echo "Found that Security Onion $INSTALLEDVERSION is currently installed." echo "" if [[ $is_airgap -eq 0 ]]; then # Let's mount the ISO since this is airgap - echo "This is airgap. Ask for a location." airgap_mounted else echo "Cloning Security Onion github repo into $UPDATE_DIR." @@ -1029,7 +1019,40 @@ EOF echo "### soup has been served at $(date) ###" } -cat << EOF +while getopts ":b:f:y" opt; do + case ${opt} in + b ) + BATCHSIZE="$OPTARG" + if ! [[ "$BATCHSIZE" =~ ^[0-9]+$ ]]; then + echo "Batch size must be a number greater than 0." + exit 1 + fi + ;; + y ) + if [[ ! -f /opt/so/state/yeselastic.txt ]]; then + echo "Cannot run soup in unattended mode. You must run soup manually to accept the Elastic License." + exit 1 + else + UNATTENDED=true + fi + ;; + f ) + ISOLOC="$OPTARG" + ;; + \? ) + echo "Usage: soup [-b] [-y] [-f ]" + exit 1 + ;; + : ) + echo "Invalid option: $OPTARG requires an argument" + exit 1 + ;; + esac +done +shift $((OPTIND - 1)) + +if [[ -z $UNATTENDED ]]; then + cat << EOF SOUP - Security Onion UPdater @@ -1041,7 +1064,9 @@ Press Enter to continue or Ctrl-C to cancel. EOF -read -r input + read -r input +fi +echo "### Preparing soup at $(date) ###" main "$@" | tee -a $SOUP_LOG From ca5339341f6f39fc91f6d3e6fb089d4e71c4b5a4 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 27 Aug 2021 11:34:28 -0400 Subject: [PATCH 18/51] Fix batch size regex to disallow 0 --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 8adabf908..60e0ce93f 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -1023,7 +1023,7 @@ while getopts ":b:f:y" opt; do case ${opt} in b ) BATCHSIZE="$OPTARG" - if ! [[ "$BATCHSIZE" =~ ^[0-9]+$ ]]; then + if ! [[ "$BATCHSIZE" =~ ^[1-9][0-9]*$ ]]; then echo "Batch size must be a number greater than 0." exit 1 fi From f8cdf5bca3987112ceb13305a124a5a2ae6a880a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 27 Aug 2021 14:39:02 -0400 Subject: [PATCH 19/51] Add Templates for all filebeat modules --- .../templates/so/so-aws-template.json.jinja | 13 +++++++++++++ .../templates/so/so-azure-template.json.jinja | 13 +++++++++++++ .../templates/so/so-barracuda-template.json.jinja | 13 +++++++++++++ .../templates/so/so-bluecoat-template.json.jinja | 13 +++++++++++++ .../templates/so/so-cef-template.json.jinja | 13 +++++++++++++ .../templates/so/so-checkpoint-template.json.jinja | 13 +++++++++++++ .../templates/so/so-cisco-template.json.jinja | 13 +++++++++++++ .../templates/so/so-cyberark-template.json.jinja | 13 +++++++++++++ .../templates/so/so-cylance-template.json.jinja | 13 +++++++++++++ .../so/so-elasticsearch-template.json.jinja | 13 +++++++++++++ .../templates/so/so-f5-template.json.jinja | 13 +++++++++++++ .../templates/so/so-fortinet-template.json.jinja | 13 +++++++++++++ .../templates/so/so-gcp-template.json.jinja | 13 +++++++++++++ .../so/so-google_workspace-template.json.jinja | 13 +++++++++++++ .../templates/so/so-imperva-template.json.jinja | 13 +++++++++++++ .../templates/so/so-infoblox-template.json.jinja | 13 +++++++++++++ .../templates/so/so-juniper-template.json.jinja | 13 +++++++++++++ .../templates/so/so-kibana-template.json.jinja | 13 +++++++++++++ .../templates/so/so-logstash-template.json.jinja | 13 +++++++++++++ .../templates/so/so-microsoft-template.json.jinja | 13 +++++++++++++ .../templates/so/so-misp-template.json.jinja | 13 +++++++++++++ .../templates/so/so-netflow-template.json.jinja | 13 +++++++++++++ .../templates/so/so-netscout-template.json.jinja | 13 +++++++++++++ .../templates/so/so-o365-template.json.jinja | 13 +++++++++++++ .../templates/so/so-okta-template.json.jinja | 13 +++++++++++++ .../templates/so/so-proofpoint-template.json.jinja | 13 +++++++++++++ .../templates/so/so-radware-template.json.jinja | 13 +++++++++++++ .../templates/so/so-redis-template.json.jinja | 13 +++++++++++++ .../templates/so/so-snort-template.json.jinja | 13 +++++++++++++ .../templates/so/so-snyk-template.json.jinja | 13 +++++++++++++ .../templates/so/so-sonicwall-template.json.jinja | 13 +++++++++++++ .../templates/so/so-sophos-template.json.jinja | 13 +++++++++++++ .../templates/so/so-squid-template.json.jinja | 13 +++++++++++++ .../templates/so/so-tomcat-template.json.jinja | 13 +++++++++++++ .../templates/so/so-zscaler-template.json.jinja | 13 +++++++++++++ 35 files changed, 455 insertions(+) create mode 100644 salt/elasticsearch/templates/so/so-aws-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-azure-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-barracuda-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-bluecoat-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-cef-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-checkpoint-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-cisco-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-cyberark-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-cylance-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-elasticsearch-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-f5-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-fortinet-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-gcp-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-google_workspace-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-imperva-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-infoblox-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-juniper-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-kibana-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-logstash-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-microsoft-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-misp-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-netflow-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-netscout-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-o365-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-okta-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-proofpoint-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-radware-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-redis-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-snort-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-snyk-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-sonicwall-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-sophos-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-squid-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-tomcat-template.json.jinja create mode 100644 salt/elasticsearch/templates/so/so-zscaler-template.json.jinja diff --git a/salt/elasticsearch/templates/so/so-aws-template.json.jinja b/salt/elasticsearch/templates/so/so-aws-template.json.jinja new file mode 100644 index 000000000..19b23dfba --- /dev/null +++ b/salt/elasticsearch/templates/so/so-aws-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-aws:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-aws:refresh', '30s') %} +{ + "index_patterns": ["so-aws-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-azure-template.json.jinja b/salt/elasticsearch/templates/so/so-azure-template.json.jinja new file mode 100644 index 000000000..51a266479 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-azure-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-azure:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-azure:refresh', '30s') %} +{ + "index_patterns": ["so-azure-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-barracuda-template.json.jinja b/salt/elasticsearch/templates/so/so-barracuda-template.json.jinja new file mode 100644 index 000000000..66967d6d1 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-barracuda-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-barracuda:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-barracuda:refresh', '30s') %} +{ + "index_patterns": ["so-barracuda-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-bluecoat-template.json.jinja b/salt/elasticsearch/templates/so/so-bluecoat-template.json.jinja new file mode 100644 index 000000000..b1714183e --- /dev/null +++ b/salt/elasticsearch/templates/so/so-bluecoat-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-bluecoat:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-bluecoat:refresh', '30s') %} +{ + "index_patterns": ["so-bluecoat-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-cef-template.json.jinja b/salt/elasticsearch/templates/so/so-cef-template.json.jinja new file mode 100644 index 000000000..0081d42e1 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-cef-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-cef:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-cef:refresh', '30s') %} +{ + "index_patterns": ["so-cef-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-checkpoint-template.json.jinja b/salt/elasticsearch/templates/so/so-checkpoint-template.json.jinja new file mode 100644 index 000000000..5d41946cf --- /dev/null +++ b/salt/elasticsearch/templates/so/so-checkpoint-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-checkpoint:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-checkpoint:refresh', '30s') %} +{ + "index_patterns": ["so-checkpoint-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-cisco-template.json.jinja b/salt/elasticsearch/templates/so/so-cisco-template.json.jinja new file mode 100644 index 000000000..e6e6d14d0 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-cisco-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-cisco:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-cisco:refresh', '30s') %} +{ + "index_patterns": ["so-cisco-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-cyberark-template.json.jinja b/salt/elasticsearch/templates/so/so-cyberark-template.json.jinja new file mode 100644 index 000000000..1647d600f --- /dev/null +++ b/salt/elasticsearch/templates/so/so-cyberark-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-cyberark:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-cyberark:refresh', '30s') %} +{ + "index_patterns": ["so-cyberark-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-cylance-template.json.jinja b/salt/elasticsearch/templates/so/so-cylance-template.json.jinja new file mode 100644 index 000000000..4ba7d0316 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-cylance-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-cylance:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-cylance:refresh', '30s') %} +{ + "index_patterns": ["so-cylance-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-elasticsearch-template.json.jinja b/salt/elasticsearch/templates/so/so-elasticsearch-template.json.jinja new file mode 100644 index 000000000..16aaaec13 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-elasticsearch-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-elasticsearch:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-elasticsearch:refresh', '30s') %} +{ + "index_patterns": ["so-elasticsearch-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-f5-template.json.jinja b/salt/elasticsearch/templates/so/so-f5-template.json.jinja new file mode 100644 index 000000000..682a37c59 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-f5-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-f5:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-f5:refresh', '30s') %} +{ + "index_patterns": ["so-f5-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-fortinet-template.json.jinja b/salt/elasticsearch/templates/so/so-fortinet-template.json.jinja new file mode 100644 index 000000000..616607f52 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-fortinet-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zeek:refresh', '30s') %} +{ + "index_patterns": ["so-zeek-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-gcp-template.json.jinja b/salt/elasticsearch/templates/so/so-gcp-template.json.jinja new file mode 100644 index 000000000..4f1db4f20 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-gcp-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-gcp:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-gcp:refresh', '30s') %} +{ + "index_patterns": ["so-gcp-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-google_workspace-template.json.jinja b/salt/elasticsearch/templates/so/so-google_workspace-template.json.jinja new file mode 100644 index 000000000..5ae26780a --- /dev/null +++ b/salt/elasticsearch/templates/so/so-google_workspace-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-google_workspace:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-google_workspace:refresh', '30s') %} +{ + "index_patterns": ["so-google_workspace-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-imperva-template.json.jinja b/salt/elasticsearch/templates/so/so-imperva-template.json.jinja new file mode 100644 index 000000000..1f574f33a --- /dev/null +++ b/salt/elasticsearch/templates/so/so-imperva-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-imperva:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-imperva:refresh', '30s') %} +{ + "index_patterns": ["so-imperva-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-infoblox-template.json.jinja b/salt/elasticsearch/templates/so/so-infoblox-template.json.jinja new file mode 100644 index 000000000..de613de7f --- /dev/null +++ b/salt/elasticsearch/templates/so/so-infoblox-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-infoblox:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-infoblox:refresh', '30s') %} +{ + "index_patterns": ["so-infoblox-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-juniper-template.json.jinja b/salt/elasticsearch/templates/so/so-juniper-template.json.jinja new file mode 100644 index 000000000..f637271a9 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-juniper-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-juniper:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-juniper:refresh', '30s') %} +{ + "index_patterns": ["so-juniper-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-kibana-template.json.jinja b/salt/elasticsearch/templates/so/so-kibana-template.json.jinja new file mode 100644 index 000000000..fe2004b0e --- /dev/null +++ b/salt/elasticsearch/templates/so/so-kibana-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-kibana:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-kibana:refresh', '30s') %} +{ + "index_patterns": ["so-kibana-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-logstash-template.json.jinja b/salt/elasticsearch/templates/so/so-logstash-template.json.jinja new file mode 100644 index 000000000..2cf0aba42 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-logstash-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-logstash:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-logstash:refresh', '30s') %} +{ + "index_patterns": ["so-logstash-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-microsoft-template.json.jinja b/salt/elasticsearch/templates/so/so-microsoft-template.json.jinja new file mode 100644 index 000000000..3493ccbb2 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-microsoft-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-microsoft:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-microsoft:refresh', '30s') %} +{ + "index_patterns": ["so-microsoft-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-misp-template.json.jinja b/salt/elasticsearch/templates/so/so-misp-template.json.jinja new file mode 100644 index 000000000..67af1efde --- /dev/null +++ b/salt/elasticsearch/templates/so/so-misp-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-misp:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-misp:refresh', '30s') %} +{ + "index_patterns": ["so-misp-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-netflow-template.json.jinja b/salt/elasticsearch/templates/so/so-netflow-template.json.jinja new file mode 100644 index 000000000..62c0972bf --- /dev/null +++ b/salt/elasticsearch/templates/so/so-netflow-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-netflow:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-netflow:refresh', '30s') %} +{ + "index_patterns": ["so-netflow-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-netscout-template.json.jinja b/salt/elasticsearch/templates/so/so-netscout-template.json.jinja new file mode 100644 index 000000000..1dfe336d9 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-netscout-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-netscout:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-netscout:refresh', '30s') %} +{ + "index_patterns": ["so-netscout-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-o365-template.json.jinja b/salt/elasticsearch/templates/so/so-o365-template.json.jinja new file mode 100644 index 000000000..c1f4826f4 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-o365-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-o365:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-o365:refresh', '30s') %} +{ + "index_patterns": ["so-o365-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-okta-template.json.jinja b/salt/elasticsearch/templates/so/so-okta-template.json.jinja new file mode 100644 index 000000000..a4f2df44e --- /dev/null +++ b/salt/elasticsearch/templates/so/so-okta-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-okta:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-okta:refresh', '30s') %} +{ + "index_patterns": ["so-okta-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-proofpoint-template.json.jinja b/salt/elasticsearch/templates/so/so-proofpoint-template.json.jinja new file mode 100644 index 000000000..d7b0ecbe2 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-proofpoint-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-proofpoint:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-proofpoint:refresh', '30s') %} +{ + "index_patterns": ["so-proofpoint-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-radware-template.json.jinja b/salt/elasticsearch/templates/so/so-radware-template.json.jinja new file mode 100644 index 000000000..4efef6a4f --- /dev/null +++ b/salt/elasticsearch/templates/so/so-radware-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-radware:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-radware:refresh', '30s') %} +{ + "index_patterns": ["so-radware-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-redis-template.json.jinja b/salt/elasticsearch/templates/so/so-redis-template.json.jinja new file mode 100644 index 000000000..616607f52 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-redis-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zeek:refresh', '30s') %} +{ + "index_patterns": ["so-zeek-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-snort-template.json.jinja b/salt/elasticsearch/templates/so/so-snort-template.json.jinja new file mode 100644 index 000000000..325f86bde --- /dev/null +++ b/salt/elasticsearch/templates/so/so-snort-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-snort:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-snort:refresh', '30s') %} +{ + "index_patterns": ["so-snort-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-snyk-template.json.jinja b/salt/elasticsearch/templates/so/so-snyk-template.json.jinja new file mode 100644 index 000000000..616607f52 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-snyk-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zeek:refresh', '30s') %} +{ + "index_patterns": ["so-zeek-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-sonicwall-template.json.jinja b/salt/elasticsearch/templates/so/so-sonicwall-template.json.jinja new file mode 100644 index 000000000..b912e4d66 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-sonicwall-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-sonicwall:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-sonicwall:refresh', '30s') %} +{ + "index_patterns": ["so-sonicwall-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-sophos-template.json.jinja b/salt/elasticsearch/templates/so/so-sophos-template.json.jinja new file mode 100644 index 000000000..616607f52 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-sophos-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zeek:refresh', '30s') %} +{ + "index_patterns": ["so-zeek-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-squid-template.json.jinja b/salt/elasticsearch/templates/so/so-squid-template.json.jinja new file mode 100644 index 000000000..616607f52 --- /dev/null +++ b/salt/elasticsearch/templates/so/so-squid-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zeek:refresh', '30s') %} +{ + "index_patterns": ["so-zeek-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-tomcat-template.json.jinja b/salt/elasticsearch/templates/so/so-tomcat-template.json.jinja new file mode 100644 index 000000000..797e71bcf --- /dev/null +++ b/salt/elasticsearch/templates/so/so-tomcat-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-tomcat:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-tomcat:refresh', '30s') %} +{ + "index_patterns": ["so-tomcat-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} diff --git a/salt/elasticsearch/templates/so/so-zscaler-template.json.jinja b/salt/elasticsearch/templates/so/so-zscaler-template.json.jinja new file mode 100644 index 000000000..01d1cab2d --- /dev/null +++ b/salt/elasticsearch/templates/so/so-zscaler-template.json.jinja @@ -0,0 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zscaler:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zscaler:refresh', '30s') %} +{ + "index_patterns": ["so-zscaler-*"], + "version":50001, + "order":11, + "settings":{ + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" + } +} From a27263435abd126f2f053dfc6bc237627e73d389 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 27 Aug 2021 14:41:04 -0400 Subject: [PATCH 20/51] Add Templates for all filebeat modules --- salt/elasticsearch/templates/so/so-snyk-template.json.jinja | 6 +++--- .../templates/so/so-sophos-template.json.jinja | 6 +++--- .../elasticsearch/templates/so/so-squid-template.json.jinja | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/salt/elasticsearch/templates/so/so-snyk-template.json.jinja b/salt/elasticsearch/templates/so/so-snyk-template.json.jinja index 616607f52..0c2d291cc 100644 --- a/salt/elasticsearch/templates/so/so-snyk-template.json.jinja +++ b/salt/elasticsearch/templates/so/so-snyk-template.json.jinja @@ -1,8 +1,8 @@ -{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:shards', 1) %} +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-snyk:shards', 1) %} {%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} -{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zeek:refresh', '30s') %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-snyk:refresh', '30s') %} { - "index_patterns": ["so-zeek-*"], + "index_patterns": ["so-snyk-*"], "version":50001, "order":11, "settings":{ diff --git a/salt/elasticsearch/templates/so/so-sophos-template.json.jinja b/salt/elasticsearch/templates/so/so-sophos-template.json.jinja index 616607f52..689e19999 100644 --- a/salt/elasticsearch/templates/so/so-sophos-template.json.jinja +++ b/salt/elasticsearch/templates/so/so-sophos-template.json.jinja @@ -1,8 +1,8 @@ -{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:shards', 1) %} +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-sophos:shards', 1) %} {%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} -{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zeek:refresh', '30s') %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-sophos:refresh', '30s') %} { - "index_patterns": ["so-zeek-*"], + "index_patterns": ["so-sophos-*"], "version":50001, "order":11, "settings":{ diff --git a/salt/elasticsearch/templates/so/so-squid-template.json.jinja b/salt/elasticsearch/templates/so/so-squid-template.json.jinja index 616607f52..9398b8a99 100644 --- a/salt/elasticsearch/templates/so/so-squid-template.json.jinja +++ b/salt/elasticsearch/templates/so/so-squid-template.json.jinja @@ -1,8 +1,8 @@ -{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:shards', 1) %} +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-squid:shards', 1) %} {%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} -{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zeek:refresh', '30s') %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-squid:refresh', '30s') %} { - "index_patterns": ["so-zeek-*"], + "index_patterns": ["so-squid-*"], "version":50001, "order":11, "settings":{ From 72859adb13b049d8e6317ae386f3fef4fb194e11 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 27 Aug 2021 15:23:01 -0400 Subject: [PATCH 21/51] Fix typo in so-checkin --- salt/common/tools/sbin/so-checkin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-checkin b/salt/common/tools/sbin/so-checkin index c70701b71..4f0583906 100755 --- a/salt/common/tools/sbin/so-checkin +++ b/salt/common/tools/sbin/so-checkin @@ -17,4 +17,4 @@ . /usr/sbin/so-common -salt-call state.highstate -linfo +salt-call state.highstate -l info From 2b34da0fee25440eaf29fd41e2c7a47af9c513ca Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 30 Aug 2021 12:32:44 -0400 Subject: [PATCH 22/51] Update HOTFIX --- HOTFIX | 1 - 1 file changed, 1 deletion(-) diff --git a/HOTFIX b/HOTFIX index 100732042..c9e34953f 100644 --- a/HOTFIX +++ b/HOTFIX @@ -1,2 +1 @@ - CURATOR GRAFANA_DASH_ALLOW From bfd632e20a6a8b5ea26caff27db08d9a61f1b49a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 30 Aug 2021 14:21:13 -0400 Subject: [PATCH 23/51] Add wazuh to exclude arg when running yum update --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 7bbaa1fda..9a64a561e 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2795,7 +2795,7 @@ update_sudoers() { update_packages() { if [ "$OS" = 'centos' ]; then yum repolist >> /dev/null - yum -y update --exclude=salt* >> "$setup_log" + yum -y update --exclude=salt*,wazuh* >> "$setup_log" else retry 50 10 "apt-get -y update" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-get -y upgrade" >> "$setup_log" 2>&1 || exit 1 From 92858cd13ab7d2e404ea45b37f67566ddd3259c2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 30 Aug 2021 17:38:29 -0400 Subject: [PATCH 24/51] Update HOTFIX --- HOTFIX | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HOTFIX b/HOTFIX index c9e34953f..e87aa6dbd 100644 --- a/HOTFIX +++ b/HOTFIX @@ -1 +1 @@ -CURATOR GRAFANA_DASH_ALLOW +CURATOR GRAFANA_DASH_ALLOW WAZUH From 4fe0a1d7b436d7393bfdac13e9251cb7002f3357 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 08:39:37 -0400 Subject: [PATCH 25/51] 2.3.70 WAZUH Hotfix sigs --- VERIFY_ISO.md | 22 +++++++++++----------- sigs/securityonion-2.3.70-WAZUH.iso.sig | Bin 0 -> 543 bytes 2 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 sigs/securityonion-2.3.70-WAZUH.iso.sig diff --git a/VERIFY_ISO.md b/VERIFY_ISO.md index 2be975430..660f1ef08 100644 --- a/VERIFY_ISO.md +++ b/VERIFY_ISO.md @@ -1,18 +1,18 @@ -### 2.3.70-GRAFANA ISO image built on 2021/08/23 +### 2.3.70-WAZUH ISO image built on 2021/08/23 ### Download and Verify -2.3.70-GRAFANA ISO image: -https://download.securityonion.net/file/securityonion/securityonion-2.3.70-GRAFANA.iso +2.3.70-WAZUH ISO image: +https://download.securityonion.net/file/securityonion/securityonion-2.3.70-WAZUH.iso -MD5: A16683FC8F2151C290E359FC6066B1F2 -SHA1: A93329C103CCCE665968F246163FBE5D41EF0510 -SHA256: 3ED0177CADF203324363916AA240A10C58DC3E9044A9ADE173A80674701A50A3 +MD5: CEDEF3C38089896C252F9E3C75F7CB15 +SHA1: FB420115C72DABDEB87C8B27F26E862C94628057 +SHA256: CC3E75A97163E9CD255DA0D9C3EB11922FA045651827F291025398943C1BC230 Signature for ISO image: -https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.70-GRAFANA.iso.sig +https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.70-WAZUH.iso.sig Signing key: https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS @@ -26,22 +26,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/ma Download the signature file for the ISO: ``` -wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.70-GRAFANA.iso.sig +wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.70-WAZUH.iso.sig ``` Download the ISO image: ``` -wget https://download.securityonion.net/file/securityonion/securityonion-2.3.70-GRAFANA.iso +wget https://download.securityonion.net/file/securityonion/securityonion-2.3.70-WAZUH.iso ``` Verify the downloaded ISO image using the signature file: ``` -gpg --verify securityonion-2.3.70-GRAFANA.iso.sig securityonion-2.3.70-GRAFANA.iso +gpg --verify securityonion-2.3.70-WAZUH.iso.sig securityonion-2.3.70-WAZUH.iso ``` The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Mon 23 Aug 2021 01:43:00 PM EDT using RSA key ID FE507013 +gpg: Signature made Mon 30 Aug 2021 06:13:14 PM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/sigs/securityonion-2.3.70-WAZUH.iso.sig b/sigs/securityonion-2.3.70-WAZUH.iso.sig new file mode 100644 index 0000000000000000000000000000000000000000..43ce74d15f27fd7404032466521b8f41493d216f GIT binary patch literal 543 zcmV+)0^t3L0vrSY0RjL91p;9$SNZ@72@re`V7LBIa1&ji5CFP43k_8LGaz5WZEq#U zjfxe1>pW9wHJSpK$w&`hIs|gd_hGkOa0Qcjg9u49g;ggxVp>DrKLfl{0N)Lu)vqNUO&}dp0)O@i(&g%Yp!t-=1v7XcqA8m- zNuGLYl+9n8BB~E5{WOI<&Yh%{#!Ri4Is5Sa+c2_#?^po`n{RAG zE>jx($L2Aywe2R-8tb>tYi`!r+iI*8O;Y#iy2!*FFHMy}aL_jo$chB>b^nUi7#~Of zXuk6R{56wMf=YM-PurQnsc0}3gEvnRvtd!qz?St+wh;T9d4Zc%9JTO{#9`lNm*1?WLA7L9 hGjwSAR+NL){>p*2%?PW}Wj1OMZdTyolF!u`t>a4(0qXz& literal 0 HcmV?d00001 From 77b82bf2c09aa6bd8bba765b552f3ce6a03a7cff Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 10:01:32 -0400 Subject: [PATCH 26/51] Update VERIFY_ISO.md --- VERIFY_ISO.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERIFY_ISO.md b/VERIFY_ISO.md index 660f1ef08..f6cc31508 100644 --- a/VERIFY_ISO.md +++ b/VERIFY_ISO.md @@ -1,4 +1,4 @@ -### 2.3.70-WAZUH ISO image built on 2021/08/23 +### 2.3.70-WAZUH ISO image built on 2021/08/30 From 9605eda55959ca74e09b8c695211251caec016c7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 10:49:39 -0400 Subject: [PATCH 27/51] Close Curators for all modules --- salt/curator/files/action/so-aws-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-azure-close.yml | 29 +++++++++++++++++++ .../files/action/so-barracuda-close.yml | 29 +++++++++++++++++++ .../files/action/so-bluecoat-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-cef-close.yml | 29 +++++++++++++++++++ .../files/action/so-checkpoint-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-cisco-close.yml | 29 +++++++++++++++++++ .../files/action/so-cyberark-close.yml | 29 +++++++++++++++++++ .../curator/files/action/so-cylance-close.yml | 29 +++++++++++++++++++ .../files/action/so-elasticsearch-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-f5-close.yml | 29 +++++++++++++++++++ .../files/action/so-fortinet-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-gcp-close.yml | 29 +++++++++++++++++++ .../action/so-google_workspace-close.yml | 29 +++++++++++++++++++ .../curator/files/action/so-imperva-close.yml | 29 +++++++++++++++++++ .../files/action/so-infoblox-close.yml | 29 +++++++++++++++++++ .../curator/files/action/so-juniper-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-kibana-close.yml | 29 +++++++++++++++++++ .../files/action/so-logstash-close.yml | 29 +++++++++++++++++++ .../files/action/so-microsoft-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-misp-close.yml | 29 +++++++++++++++++++ .../curator/files/action/so-netflow-close.yml | 29 +++++++++++++++++++ .../files/action/so-netscout-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-o365-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-okta-close.yml | 29 +++++++++++++++++++ .../files/action/so-proofpoint-close.yml | 29 +++++++++++++++++++ .../curator/files/action/so-radware-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-redis-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-snort-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-snyk-close.yml | 29 +++++++++++++++++++ .../files/action/so-sonicwall-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-sophos-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-tomcat-close.yml | 29 +++++++++++++++++++ .../curator/files/action/so-zscaler-close.yml | 29 +++++++++++++++++++ 34 files changed, 986 insertions(+) create mode 100644 salt/curator/files/action/so-aws-close.yml create mode 100644 salt/curator/files/action/so-azure-close.yml create mode 100644 salt/curator/files/action/so-barracuda-close.yml create mode 100644 salt/curator/files/action/so-bluecoat-close.yml create mode 100644 salt/curator/files/action/so-cef-close.yml create mode 100644 salt/curator/files/action/so-checkpoint-close.yml create mode 100644 salt/curator/files/action/so-cisco-close.yml create mode 100644 salt/curator/files/action/so-cyberark-close.yml create mode 100644 salt/curator/files/action/so-cylance-close.yml create mode 100644 salt/curator/files/action/so-elasticsearch-close.yml create mode 100644 salt/curator/files/action/so-f5-close.yml create mode 100644 salt/curator/files/action/so-fortinet-close.yml create mode 100644 salt/curator/files/action/so-gcp-close.yml create mode 100644 salt/curator/files/action/so-google_workspace-close.yml create mode 100644 salt/curator/files/action/so-imperva-close.yml create mode 100644 salt/curator/files/action/so-infoblox-close.yml create mode 100644 salt/curator/files/action/so-juniper-close.yml create mode 100644 salt/curator/files/action/so-kibana-close.yml create mode 100644 salt/curator/files/action/so-logstash-close.yml create mode 100644 salt/curator/files/action/so-microsoft-close.yml create mode 100644 salt/curator/files/action/so-misp-close.yml create mode 100644 salt/curator/files/action/so-netflow-close.yml create mode 100644 salt/curator/files/action/so-netscout-close.yml create mode 100644 salt/curator/files/action/so-o365-close.yml create mode 100644 salt/curator/files/action/so-okta-close.yml create mode 100644 salt/curator/files/action/so-proofpoint-close.yml create mode 100644 salt/curator/files/action/so-radware-close.yml create mode 100644 salt/curator/files/action/so-redis-close.yml create mode 100644 salt/curator/files/action/so-snort-close.yml create mode 100644 salt/curator/files/action/so-snyk-close.yml create mode 100644 salt/curator/files/action/so-sonicwall-close.yml create mode 100644 salt/curator/files/action/so-sophos-close.yml create mode 100644 salt/curator/files/action/so-tomcat-close.yml create mode 100644 salt/curator/files/action/so-zscaler-close.yml diff --git a/salt/curator/files/action/so-aws-close.yml b/salt/curator/files/action/so-aws-close.yml new file mode 100644 index 000000000..44f0bfa97 --- /dev/null +++ b/salt/curator/files/action/so-aws-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-aws:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close aws indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-aws.*|so-aws.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-azure-close.yml b/salt/curator/files/action/so-azure-close.yml new file mode 100644 index 000000000..901b2c0ba --- /dev/null +++ b/salt/curator/files/action/so-azure-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-azure:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close azure indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-azure.*|so-azure.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-barracuda-close.yml b/salt/curator/files/action/so-barracuda-close.yml new file mode 100644 index 000000000..496832db7 --- /dev/null +++ b/salt/curator/files/action/so-barracuda-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-barracuda:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close barracuda indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-barracuda.*|so-barracuda.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-bluecoat-close.yml b/salt/curator/files/action/so-bluecoat-close.yml new file mode 100644 index 000000000..86d9277eb --- /dev/null +++ b/salt/curator/files/action/so-bluecoat-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-bluecoat:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close bluecoat indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-bluecoat.*|so-bluecoat.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-cef-close.yml b/salt/curator/files/action/so-cef-close.yml new file mode 100644 index 000000000..49e07f764 --- /dev/null +++ b/salt/curator/files/action/so-cef-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-cef:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close cef indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-cef.*|so-cef.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-checkpoint-close.yml b/salt/curator/files/action/so-checkpoint-close.yml new file mode 100644 index 000000000..cffdf6473 --- /dev/null +++ b/salt/curator/files/action/so-checkpoint-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-checkpoint:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close checkpoint indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-checkpoint.*|so-checkpoint.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-cisco-close.yml b/salt/curator/files/action/so-cisco-close.yml new file mode 100644 index 000000000..cd1faade1 --- /dev/null +++ b/salt/curator/files/action/so-cisco-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-cisco:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close cisco indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-cisco.*|so-cisco.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-cyberark-close.yml b/salt/curator/files/action/so-cyberark-close.yml new file mode 100644 index 000000000..e352e8355 --- /dev/null +++ b/salt/curator/files/action/so-cyberark-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-cyberark:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close cyberark indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-cyberark.*|so-cyberark.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-cylance-close.yml b/salt/curator/files/action/so-cylance-close.yml new file mode 100644 index 000000000..d808569fb --- /dev/null +++ b/salt/curator/files/action/so-cylance-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-cylance:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close cylance indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-cylance.*|so-cylance.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-elasticsearch-close.yml b/salt/curator/files/action/so-elasticsearch-close.yml new file mode 100644 index 000000000..3c4ff0dac --- /dev/null +++ b/salt/curator/files/action/so-elasticsearch-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-elasticsearch:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close elasticsearch indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-elasticsearch.*|so-elasticsearch.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-f5-close.yml b/salt/curator/files/action/so-f5-close.yml new file mode 100644 index 000000000..e1cdb48a1 --- /dev/null +++ b/salt/curator/files/action/so-f5-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-f5:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close f5 indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-f5.*|so-f5.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-fortinet-close.yml b/salt/curator/files/action/so-fortinet-close.yml new file mode 100644 index 000000000..e11fb86c6 --- /dev/null +++ b/salt/curator/files/action/so-fortinet-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-fortinet:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close fortinet indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-fortinet.*|so-fortinet.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-gcp-close.yml b/salt/curator/files/action/so-gcp-close.yml new file mode 100644 index 000000000..f9dd0af24 --- /dev/null +++ b/salt/curator/files/action/so-gcp-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-gcp:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close gcp indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-gcp.*|so-gcp.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-google_workspace-close.yml b/salt/curator/files/action/so-google_workspace-close.yml new file mode 100644 index 000000000..1ecda5893 --- /dev/null +++ b/salt/curator/files/action/so-google_workspace-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-google_workspace:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close google_workspace indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-google_workspace.*|so-google_workspace.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-imperva-close.yml b/salt/curator/files/action/so-imperva-close.yml new file mode 100644 index 000000000..55ec2e472 --- /dev/null +++ b/salt/curator/files/action/so-imperva-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-imperva:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close imperva indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-imperva.*|so-imperva.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-infoblox-close.yml b/salt/curator/files/action/so-infoblox-close.yml new file mode 100644 index 000000000..9fd4c5070 --- /dev/null +++ b/salt/curator/files/action/so-infoblox-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-infoblox:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close infoblox indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-infoblox.*|so-infoblox.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-juniper-close.yml b/salt/curator/files/action/so-juniper-close.yml new file mode 100644 index 000000000..466a51eca --- /dev/null +++ b/salt/curator/files/action/so-juniper-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-juniper:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close juniper indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-juniper.*|so-juniper.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-kibana-close.yml b/salt/curator/files/action/so-kibana-close.yml new file mode 100644 index 000000000..05583d853 --- /dev/null +++ b/salt/curator/files/action/so-kibana-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close IDS indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-ids.*|so-ids.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-logstash-close.yml b/salt/curator/files/action/so-logstash-close.yml new file mode 100644 index 000000000..23787e237 --- /dev/null +++ b/salt/curator/files/action/so-logstash-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-logstash:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close logstash indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-logstash.*|so-logstash.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-microsoft-close.yml b/salt/curator/files/action/so-microsoft-close.yml new file mode 100644 index 000000000..f4eaf738f --- /dev/null +++ b/salt/curator/files/action/so-microsoft-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-microsoft:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close microsoft indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-microsoft.*|so-microsoft.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-misp-close.yml b/salt/curator/files/action/so-misp-close.yml new file mode 100644 index 000000000..05583d853 --- /dev/null +++ b/salt/curator/files/action/so-misp-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close IDS indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-ids.*|so-ids.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-netflow-close.yml b/salt/curator/files/action/so-netflow-close.yml new file mode 100644 index 000000000..cc9ade28d --- /dev/null +++ b/salt/curator/files/action/so-netflow-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-netflow:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close netflow indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-netflow.*|so-netflow.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-netscout-close.yml b/salt/curator/files/action/so-netscout-close.yml new file mode 100644 index 000000000..05583d853 --- /dev/null +++ b/salt/curator/files/action/so-netscout-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close IDS indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-ids.*|so-ids.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-o365-close.yml b/salt/curator/files/action/so-o365-close.yml new file mode 100644 index 000000000..4dece060f --- /dev/null +++ b/salt/curator/files/action/so-o365-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-o365:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close o365 indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-o365.*|so-o365.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-okta-close.yml b/salt/curator/files/action/so-okta-close.yml new file mode 100644 index 000000000..10f7e4b60 --- /dev/null +++ b/salt/curator/files/action/so-okta-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-okta:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close okta indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-okta.*|so-okta.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-proofpoint-close.yml b/salt/curator/files/action/so-proofpoint-close.yml new file mode 100644 index 000000000..888c9fc64 --- /dev/null +++ b/salt/curator/files/action/so-proofpoint-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-proofpoint:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close proofpoint indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-proofpoint.*|so-proofpoint.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-radware-close.yml b/salt/curator/files/action/so-radware-close.yml new file mode 100644 index 000000000..05583d853 --- /dev/null +++ b/salt/curator/files/action/so-radware-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close IDS indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-ids.*|so-ids.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-redis-close.yml b/salt/curator/files/action/so-redis-close.yml new file mode 100644 index 000000000..b69935f21 --- /dev/null +++ b/salt/curator/files/action/so-redis-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-redis:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close redis indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-redis.*|so-redis.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-snort-close.yml b/salt/curator/files/action/so-snort-close.yml new file mode 100644 index 000000000..8f6209255 --- /dev/null +++ b/salt/curator/files/action/so-snort-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-snort:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close snort indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-snort.*|so-snort.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-snyk-close.yml b/salt/curator/files/action/so-snyk-close.yml new file mode 100644 index 000000000..05583d853 --- /dev/null +++ b/salt/curator/files/action/so-snyk-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close IDS indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-ids.*|so-ids.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-sonicwall-close.yml b/salt/curator/files/action/so-sonicwall-close.yml new file mode 100644 index 000000000..9cc23d3af --- /dev/null +++ b/salt/curator/files/action/so-sonicwall-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-sonicwall:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close sonicwall indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-sonicwall.*|so-sonicwall.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-sophos-close.yml b/salt/curator/files/action/so-sophos-close.yml new file mode 100644 index 000000000..b7574b996 --- /dev/null +++ b/salt/curator/files/action/so-sophos-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-sophos:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close sophos indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-sophos.*|so-sophos.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-tomcat-close.yml b/salt/curator/files/action/so-tomcat-close.yml new file mode 100644 index 000000000..ea0d95b0a --- /dev/null +++ b/salt/curator/files/action/so-tomcat-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-tomcat:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close tomcat indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-tomcat.*|so-tomcat.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-zscaler-close.yml b/salt/curator/files/action/so-zscaler-close.yml new file mode 100644 index 000000000..5a008a27d --- /dev/null +++ b/salt/curator/files/action/so-zscaler-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-zscaler:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close zscaler indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-zscaler.*|so-zscaler.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: From 3425a0fe78571b493495017d1df32a93e56edc19 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 11:12:21 -0400 Subject: [PATCH 28/51] Delete Curators for all modules --- salt/curator/files/action/so-aws-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-azure-delete.yml | 29 +++++++++++++++++++ .../files/action/so-barracuda-delete.yml | 29 +++++++++++++++++++ .../files/action/so-bluecoat-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-cef-delete.yml | 29 +++++++++++++++++++ .../files/action/so-checkpoint-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-cisco-delete.yml | 29 +++++++++++++++++++ .../files/action/so-cyberark-delete.yml | 29 +++++++++++++++++++ .../files/action/so-cylance-delete.yml | 29 +++++++++++++++++++ .../files/action/so-elasticsearch-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-f5-delete.yml | 29 +++++++++++++++++++ .../files/action/so-fortinet-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-gcp-delete.yml | 29 +++++++++++++++++++ .../files/action/so-google_workspace.yml | 29 +++++++++++++++++++ .../files/action/so-imperva-delete.yml | 29 +++++++++++++++++++ .../files/action/so-infoblox-delete.yml | 29 +++++++++++++++++++ .../files/action/so-juniper-delete.yml | 29 +++++++++++++++++++ .../curator/files/action/so-kibana-delete.yml | 29 +++++++++++++++++++ .../files/action/so-logstash-delete.yml | 29 +++++++++++++++++++ .../files/action/so-microsoft-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-misp-close.yml | 6 ++-- salt/curator/files/action/so-misp-delete.yml | 29 +++++++++++++++++++ .../files/action/so-netflow-delete.yml | 29 +++++++++++++++++++ .../files/action/so-netscout-close.yml | 6 ++-- salt/curator/files/action/so-o365-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-okta.delete.yml | 29 +++++++++++++++++++ .../files/action/so-proofpoint-delete.yml | 29 +++++++++++++++++++ .../curator/files/action/so-radware-close.yml | 6 ++-- .../files/action/so-radware-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-redis-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-snort-delete.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-snyk-close.yml | 6 ++-- salt/curator/files/action/so-snyk-delete.yml | 29 +++++++++++++++++++ .../files/action/so-sonicwall-delete.yml | 29 +++++++++++++++++++ .../curator/files/action/so-sophos-delete.yml | 29 +++++++++++++++++++ .../curator/files/action/so-tomcat-delete.yml | 29 +++++++++++++++++++ .../files/action/so-zscaler-delete.yml | 29 +++++++++++++++++++ 37 files changed, 969 insertions(+), 12 deletions(-) create mode 100644 salt/curator/files/action/so-aws-delete.yml create mode 100644 salt/curator/files/action/so-azure-delete.yml create mode 100644 salt/curator/files/action/so-barracuda-delete.yml create mode 100644 salt/curator/files/action/so-bluecoat-delete.yml create mode 100644 salt/curator/files/action/so-cef-delete.yml create mode 100644 salt/curator/files/action/so-checkpoint-delete.yml create mode 100644 salt/curator/files/action/so-cisco-delete.yml create mode 100644 salt/curator/files/action/so-cyberark-delete.yml create mode 100644 salt/curator/files/action/so-cylance-delete.yml create mode 100644 salt/curator/files/action/so-elasticsearch-delete.yml create mode 100644 salt/curator/files/action/so-f5-delete.yml create mode 100644 salt/curator/files/action/so-fortinet-delete.yml create mode 100644 salt/curator/files/action/so-gcp-delete.yml create mode 100644 salt/curator/files/action/so-google_workspace.yml create mode 100644 salt/curator/files/action/so-imperva-delete.yml create mode 100644 salt/curator/files/action/so-infoblox-delete.yml create mode 100644 salt/curator/files/action/so-juniper-delete.yml create mode 100644 salt/curator/files/action/so-kibana-delete.yml create mode 100644 salt/curator/files/action/so-logstash-delete.yml create mode 100644 salt/curator/files/action/so-microsoft-delete.yml create mode 100644 salt/curator/files/action/so-misp-delete.yml create mode 100644 salt/curator/files/action/so-netflow-delete.yml create mode 100644 salt/curator/files/action/so-o365-delete.yml create mode 100644 salt/curator/files/action/so-okta.delete.yml create mode 100644 salt/curator/files/action/so-proofpoint-delete.yml create mode 100644 salt/curator/files/action/so-radware-delete.yml create mode 100644 salt/curator/files/action/so-redis-delete.yml create mode 100644 salt/curator/files/action/so-snort-delete.yml create mode 100644 salt/curator/files/action/so-snyk-delete.yml create mode 100644 salt/curator/files/action/so-sonicwall-delete.yml create mode 100644 salt/curator/files/action/so-sophos-delete.yml create mode 100644 salt/curator/files/action/so-tomcat-delete.yml create mode 100644 salt/curator/files/action/so-zscaler-delete.yml diff --git a/salt/curator/files/action/so-aws-delete.yml b/salt/curator/files/action/so-aws-delete.yml new file mode 100644 index 000000000..a67ee88b8 --- /dev/null +++ b/salt/curator/files/action/so-aws-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete aws indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-aws.*|so-aws.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-azure-delete.yml b/salt/curator/files/action/so-azure-delete.yml new file mode 100644 index 000000000..102a69d3d --- /dev/null +++ b/salt/curator/files/action/so-azure-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-azure:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete azure indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-azure.*|so-azure.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-barracuda-delete.yml b/salt/curator/files/action/so-barracuda-delete.yml new file mode 100644 index 000000000..49d472618 --- /dev/null +++ b/salt/curator/files/action/so-barracuda-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-barracuda:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete barracuda indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-barracuda.*|so-barracuda.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-bluecoat-delete.yml b/salt/curator/files/action/so-bluecoat-delete.yml new file mode 100644 index 000000000..318624416 --- /dev/null +++ b/salt/curator/files/action/so-bluecoat-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-bluecoat:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete bluecoat indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-bluecoat.*|so-bluecoat.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-cef-delete.yml b/salt/curator/files/action/so-cef-delete.yml new file mode 100644 index 000000000..0ee7d6501 --- /dev/null +++ b/salt/curator/files/action/so-cef-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-cef:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete cef indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-cef.*|so-cef.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-checkpoint-delete.yml b/salt/curator/files/action/so-checkpoint-delete.yml new file mode 100644 index 000000000..d1ac13efe --- /dev/null +++ b/salt/curator/files/action/so-checkpoint-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-checkpoint:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete checkpoint indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-checkpoint.*|so-checkpoint.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-cisco-delete.yml b/salt/curator/files/action/so-cisco-delete.yml new file mode 100644 index 000000000..bb5e06f7f --- /dev/null +++ b/salt/curator/files/action/so-cisco-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-cisco:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete cisco indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-cisco.*|so-cisco.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-cyberark-delete.yml b/salt/curator/files/action/so-cyberark-delete.yml new file mode 100644 index 000000000..784f6881e --- /dev/null +++ b/salt/curator/files/action/so-cyberark-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-cyberark:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete cyberark indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-cyberark.*|so-cyberark.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-cylance-delete.yml b/salt/curator/files/action/so-cylance-delete.yml new file mode 100644 index 000000000..54cf3938b --- /dev/null +++ b/salt/curator/files/action/so-cylance-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-cylance:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete cylance indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-cylance.*|so-cylance.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-elasticsearch-delete.yml b/salt/curator/files/action/so-elasticsearch-delete.yml new file mode 100644 index 000000000..05cc68abe --- /dev/null +++ b/salt/curator/files/action/so-elasticsearch-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-elasticsearch:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete elasticsearch indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-elasticsearch.*|so-elasticsearch.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-f5-delete.yml b/salt/curator/files/action/so-f5-delete.yml new file mode 100644 index 000000000..06704010a --- /dev/null +++ b/salt/curator/files/action/so-f5-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-f5:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete f5 indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-f5.*|so-f5.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-fortinet-delete.yml b/salt/curator/files/action/so-fortinet-delete.yml new file mode 100644 index 000000000..9379e47c2 --- /dev/null +++ b/salt/curator/files/action/so-fortinet-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-fortinet:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete fortinet indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-fortinet.*|so-fortinet.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-gcp-delete.yml b/salt/curator/files/action/so-gcp-delete.yml new file mode 100644 index 000000000..5c8ab33d8 --- /dev/null +++ b/salt/curator/files/action/so-gcp-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-gcp:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete gcp indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-gcp.*|so-gcp.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-google_workspace.yml b/salt/curator/files/action/so-google_workspace.yml new file mode 100644 index 000000000..923feda8f --- /dev/null +++ b/salt/curator/files/action/so-google_workspace.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-google_workspace:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete google_workspace indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-google_workspace.*|so-google_workspace.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-imperva-delete.yml b/salt/curator/files/action/so-imperva-delete.yml new file mode 100644 index 000000000..b5526e2fb --- /dev/null +++ b/salt/curator/files/action/so-imperva-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-imperva:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete imperva indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-imperva.*|so-imperva.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-infoblox-delete.yml b/salt/curator/files/action/so-infoblox-delete.yml new file mode 100644 index 000000000..0a7fdafbe --- /dev/null +++ b/salt/curator/files/action/so-infoblox-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-infoblox:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete infoblox indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-infoblox.*|so-infoblox.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-juniper-delete.yml b/salt/curator/files/action/so-juniper-delete.yml new file mode 100644 index 000000000..18abc86ac --- /dev/null +++ b/salt/curator/files/action/so-juniper-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-juniper:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete juniper indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-juniper.*|so-juniper.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-kibana-delete.yml b/salt/curator/files/action/so-kibana-delete.yml new file mode 100644 index 000000000..5a775b8de --- /dev/null +++ b/salt/curator/files/action/so-kibana-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-kibana:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete kibana indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-kibana.*|so-kibana.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-logstash-delete.yml b/salt/curator/files/action/so-logstash-delete.yml new file mode 100644 index 000000000..d9ff848da --- /dev/null +++ b/salt/curator/files/action/so-logstash-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-logstash:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete logstash indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-logstash.*|so-logstash.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-microsoft-delete.yml b/salt/curator/files/action/so-microsoft-delete.yml new file mode 100644 index 000000000..f1a854c83 --- /dev/null +++ b/salt/curator/files/action/so-microsoft-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-microsoft:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete microsoft indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-microsoft.*|so-microsoft.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-misp-close.yml b/salt/curator/files/action/so-misp-close.yml index 05583d853..e39781353 100644 --- a/salt/curator/files/action/so-misp-close.yml +++ b/salt/curator/files/action/so-misp-close.yml @@ -1,4 +1,4 @@ -{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-misp:close', 30) -%} --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" @@ -10,7 +10,7 @@ actions: 1: action: close description: >- - Close IDS indices older than {{cur_close_days}} days. + Close misp indices older than {{cur_close_days}} days. options: delete_aliases: False timeout_override: @@ -19,7 +19,7 @@ actions: filters: - filtertype: pattern kind: regex - value: '^(logstash-ids.*|so-ids.*)$' + value: '^(logstash-misp.*|so-misp.*)$' - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-misp-delete.yml b/salt/curator/files/action/so-misp-delete.yml new file mode 100644 index 000000000..ceaa9c73d --- /dev/null +++ b/salt/curator/files/action/so-misp-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-misp:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete misp indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-misp.*|so-misp.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-netflow-delete.yml b/salt/curator/files/action/so-netflow-delete.yml new file mode 100644 index 000000000..5bc76ad15 --- /dev/null +++ b/salt/curator/files/action/so-netflow-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-netflow:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete netflow indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-netflow.*|so-netflow.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-netscout-close.yml b/salt/curator/files/action/so-netscout-close.yml index 05583d853..d99374d2f 100644 --- a/salt/curator/files/action/so-netscout-close.yml +++ b/salt/curator/files/action/so-netscout-close.yml @@ -1,4 +1,4 @@ -{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-netscout:close', 30) -%} --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" @@ -10,7 +10,7 @@ actions: 1: action: close description: >- - Close IDS indices older than {{cur_close_days}} days. + Close netscout indices older than {{cur_close_days}} days. options: delete_aliases: False timeout_override: @@ -19,7 +19,7 @@ actions: filters: - filtertype: pattern kind: regex - value: '^(logstash-ids.*|so-ids.*)$' + value: '^(logstash-netscout.*|so-netscout.*)$' - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-o365-delete.yml b/salt/curator/files/action/so-o365-delete.yml new file mode 100644 index 000000000..13c7c1344 --- /dev/null +++ b/salt/curator/files/action/so-o365-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-o365:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete o365 indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-o365.*|so-o365.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-okta.delete.yml b/salt/curator/files/action/so-okta.delete.yml new file mode 100644 index 000000000..1beeb0fc0 --- /dev/null +++ b/salt/curator/files/action/so-okta.delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-okta:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete okta indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-okta.*|so-okta.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-proofpoint-delete.yml b/salt/curator/files/action/so-proofpoint-delete.yml new file mode 100644 index 000000000..903dde204 --- /dev/null +++ b/salt/curator/files/action/so-proofpoint-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-proofpoint:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete proofpoint indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-proofpoint.*|so-proofpoint.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-radware-close.yml b/salt/curator/files/action/so-radware-close.yml index 05583d853..59a7bbafd 100644 --- a/salt/curator/files/action/so-radware-close.yml +++ b/salt/curator/files/action/so-radware-close.yml @@ -1,4 +1,4 @@ -{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-radware:close', 30) -%} --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" @@ -10,7 +10,7 @@ actions: 1: action: close description: >- - Close IDS indices older than {{cur_close_days}} days. + Close radware indices older than {{cur_close_days}} days. options: delete_aliases: False timeout_override: @@ -19,7 +19,7 @@ actions: filters: - filtertype: pattern kind: regex - value: '^(logstash-ids.*|so-ids.*)$' + value: '^(logstash-radware.*|so-radware.*)$' - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-radware-delete.yml b/salt/curator/files/action/so-radware-delete.yml new file mode 100644 index 000000000..1fe09cded --- /dev/null +++ b/salt/curator/files/action/so-radware-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-radware:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete radware indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-radware.*|so-radware.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-redis-delete.yml b/salt/curator/files/action/so-redis-delete.yml new file mode 100644 index 000000000..f6e73dce8 --- /dev/null +++ b/salt/curator/files/action/so-redis-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-redis:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete redis indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-redis.*|so-redis.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-snort-delete.yml b/salt/curator/files/action/so-snort-delete.yml new file mode 100644 index 000000000..50f68988b --- /dev/null +++ b/salt/curator/files/action/so-snort-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-snort:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete snort indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-snort.*|so-snort.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-snyk-close.yml b/salt/curator/files/action/so-snyk-close.yml index 05583d853..e13d8f98d 100644 --- a/salt/curator/files/action/so-snyk-close.yml +++ b/salt/curator/files/action/so-snyk-close.yml @@ -1,4 +1,4 @@ -{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-snyk:close', 30) -%} --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" @@ -10,7 +10,7 @@ actions: 1: action: close description: >- - Close IDS indices older than {{cur_close_days}} days. + Close snyk indices older than {{cur_close_days}} days. options: delete_aliases: False timeout_override: @@ -19,7 +19,7 @@ actions: filters: - filtertype: pattern kind: regex - value: '^(logstash-ids.*|so-ids.*)$' + value: '^(logstash-snyk.*|so-snyk.*)$' - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-snyk-delete.yml b/salt/curator/files/action/so-snyk-delete.yml new file mode 100644 index 000000000..cec0b942f --- /dev/null +++ b/salt/curator/files/action/so-snyk-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-snyk:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete snyk indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-snyk.*|so-snyk.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-sonicwall-delete.yml b/salt/curator/files/action/so-sonicwall-delete.yml new file mode 100644 index 000000000..c7d38361f --- /dev/null +++ b/salt/curator/files/action/so-sonicwall-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-sonicwall:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete sonicwall indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-sonicwall.*|so-sonicwall.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-sophos-delete.yml b/salt/curator/files/action/so-sophos-delete.yml new file mode 100644 index 000000000..433df908a --- /dev/null +++ b/salt/curator/files/action/so-sophos-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-sophos:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete sophos indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-sophos.*|so-sophos.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-tomcat-delete.yml b/salt/curator/files/action/so-tomcat-delete.yml new file mode 100644 index 000000000..77035613f --- /dev/null +++ b/salt/curator/files/action/so-tomcat-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-tomcat:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete tomcat indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-tomcat.*|so-tomcat.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-zscaler-delete.yml b/salt/curator/files/action/so-zscaler-delete.yml new file mode 100644 index 000000000..238fea083 --- /dev/null +++ b/salt/curator/files/action/so-zscaler-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-zscaler:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete zscaler indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-zscaler.*|so-zscaler.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file From c09cebbd6b9f46334f70b633166e99620f2f19f8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 13:42:11 -0400 Subject: [PATCH 29/51] Add Actions for close and delete in cluster mode --- .../files/bin/so-curator-cluster-close | 44 +++++++++++++++++++ .../files/bin/so-curator-cluster-delete | 44 +++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 salt/curator/files/bin/so-curator-cluster-close create mode 100644 salt/curator/files/bin/so-curator-cluster-delete diff --git a/salt/curator/files/bin/so-curator-cluster-close b/salt/curator/files/bin/so-curator-cluster-close new file mode 100644 index 000000000..1a4124b53 --- /dev/null +++ b/salt/curator/files/bin/so-curator-cluster-close @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +APP=close +lf=/tmp/$APP-pidLockFile +# create empty lock file if none exists +cat /dev/null >> $lf +read lastPID < $lf +# if lastPID is not null and a process with that pid exists , exit +[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit +echo $$ > $lf + +{% from 'filebeat/map.jinja' import THIRDPARTY with context %} +{% from 'filebeat/map.jinja' import SO with context %} + +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-firewall-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ids-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-import-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-osquery-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-close.yml > /dev/null 2>&1; +{% for INDEX in THIRDPARTY -%} +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1; +{% done -%} +{% for INDEX in SO -%} +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} +{% done -%} \ No newline at end of file diff --git a/salt/curator/files/bin/so-curator-cluster-delete b/salt/curator/files/bin/so-curator-cluster-delete new file mode 100644 index 000000000..cccb270b0 --- /dev/null +++ b/salt/curator/files/bin/so-curator-cluster-delete @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +APP=delete +lf=/tmp/$APP-pidLockFile +# create empty lock file if none exists +cat /dev/null >> $lf +read lastPID < $lf +# if lastPID is not null and a process with that pid exists , exit +[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit +echo $$ > $lf + +{% from 'filebeat/map.jinja' import THIRDPARTY with context %} +{% from 'filebeat/map.jinja' import SO with context %} + +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-firewall-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ids-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-import-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-osquery-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-delete.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-delete.yml > /dev/null 2>&1; +{% for INDEX in THIRDPARTY -%} +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-delete.yml > /dev/null 2>&1; +{% done -%} +{% for INDEX in SO -%} +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-delete.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} +{% done -%} \ No newline at end of file From 476ecccbc10c8b0a414e2c1d060821eeccd6257d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 15:08:10 -0400 Subject: [PATCH 30/51] Add Actions for warm indices --- salt/curator/files/action/so-aws-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-azure-warm.yml | 24 +++++++++++++++ .../files/action/so-barracuda-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-beats-warm.yml | 24 +++++++++++++++ .../curator/files/action/so-bluecoat-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-cef-warm.yml | 24 +++++++++++++++ .../files/action/so-checkpoint-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-cisco-warm.yml | 24 +++++++++++++++ .../curator/files/action/so-cyberark-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-cylance-warm.yml | 24 +++++++++++++++ .../files/action/so-elasticsearch-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-f5-warm.yml | 24 +++++++++++++++ .../curator/files/action/so-firewall-warm.yml | 24 +++++++++++++++ .../curator/files/action/so-fortinet-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-gcp-warm.yml | 24 +++++++++++++++ ...ace.yml => so-google_workspace-delete.yml} | 0 .../files/action/so-google_workspace-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-ids-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-imperva-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-import-warm.yml | 24 +++++++++++++++ .../curator/files/action/so-infoblox-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-juniper-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-kibana-warm.yml | 24 +++++++++++++++ .../curator/files/action/so-logstash-warm.yml | 24 +++++++++++++++ .../files/action/so-microsoft-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-misp-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-netflow-warm.yml | 24 +++++++++++++++ .../files/action/so-netscout-delete.yml | 29 +++++++++++++++++++ .../curator/files/action/so-netscout-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-okta-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-osquery-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-ossec-warm.yml | 24 +++++++++++++++ .../files/action/so-proofpoint-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-radware-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-redis-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-snort-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-snyk-warm.yml | 24 +++++++++++++++ .../files/action/so-sonicwall-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-sophos-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-strelka-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-syslog-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-tomcat-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-zeek-warm.yml | 24 +++++++++++++++ salt/curator/files/action/so-zscaler-warm.yml | 24 +++++++++++++++ salt/curator/files/action/warm.yml | 23 --------------- salt/curator/files/bin/so-cluster-warm | 27 +++++++++++++++++ 46 files changed, 1064 insertions(+), 23 deletions(-) create mode 100644 salt/curator/files/action/so-aws-warm.yml create mode 100644 salt/curator/files/action/so-azure-warm.yml create mode 100644 salt/curator/files/action/so-barracuda-warm.yml create mode 100644 salt/curator/files/action/so-beats-warm.yml create mode 100644 salt/curator/files/action/so-bluecoat-warm.yml create mode 100644 salt/curator/files/action/so-cef-warm.yml create mode 100644 salt/curator/files/action/so-checkpoint-warm.yml create mode 100644 salt/curator/files/action/so-cisco-warm.yml create mode 100644 salt/curator/files/action/so-cyberark-warm.yml create mode 100644 salt/curator/files/action/so-cylance-warm.yml create mode 100644 salt/curator/files/action/so-elasticsearch-warm.yml create mode 100644 salt/curator/files/action/so-f5-warm.yml create mode 100644 salt/curator/files/action/so-firewall-warm.yml create mode 100644 salt/curator/files/action/so-fortinet-warm.yml create mode 100644 salt/curator/files/action/so-gcp-warm.yml rename salt/curator/files/action/{so-google_workspace.yml => so-google_workspace-delete.yml} (100%) create mode 100644 salt/curator/files/action/so-google_workspace-warm.yml create mode 100644 salt/curator/files/action/so-ids-warm.yml create mode 100644 salt/curator/files/action/so-imperva-warm.yml create mode 100644 salt/curator/files/action/so-import-warm.yml create mode 100644 salt/curator/files/action/so-infoblox-warm.yml create mode 100644 salt/curator/files/action/so-juniper-warm.yml create mode 100644 salt/curator/files/action/so-kibana-warm.yml create mode 100644 salt/curator/files/action/so-logstash-warm.yml create mode 100644 salt/curator/files/action/so-microsoft-warm.yml create mode 100644 salt/curator/files/action/so-misp-warm.yml create mode 100644 salt/curator/files/action/so-netflow-warm.yml create mode 100644 salt/curator/files/action/so-netscout-delete.yml create mode 100644 salt/curator/files/action/so-netscout-warm.yml create mode 100644 salt/curator/files/action/so-okta-warm.yml create mode 100644 salt/curator/files/action/so-osquery-warm.yml create mode 100644 salt/curator/files/action/so-ossec-warm.yml create mode 100644 salt/curator/files/action/so-proofpoint-warm.yml create mode 100644 salt/curator/files/action/so-radware-warm.yml create mode 100644 salt/curator/files/action/so-redis-warm.yml create mode 100644 salt/curator/files/action/so-snort-warm.yml create mode 100644 salt/curator/files/action/so-snyk-warm.yml create mode 100644 salt/curator/files/action/so-sonicwall-warm.yml create mode 100644 salt/curator/files/action/so-sophos-warm.yml create mode 100644 salt/curator/files/action/so-strelka-warm.yml create mode 100644 salt/curator/files/action/so-syslog-warm.yml create mode 100644 salt/curator/files/action/so-tomcat-warm.yml create mode 100644 salt/curator/files/action/so-zeek-warm.yml create mode 100644 salt/curator/files/action/so-zscaler-warm.yml delete mode 100644 salt/curator/files/action/warm.yml create mode 100644 salt/curator/files/bin/so-cluster-warm diff --git a/salt/curator/files/action/so-aws-warm.yml b/salt/curator/files/action/so-aws-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-aws-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-azure-warm.yml b/salt/curator/files/action/so-azure-warm.yml new file mode 100644 index 000000000..d6f606125 --- /dev/null +++ b/salt/curator/files/action/so-azure-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-azure:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-azure + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-barracuda-warm.yml b/salt/curator/files/action/so-barracuda-warm.yml new file mode 100644 index 000000000..334a4114a --- /dev/null +++ b/salt/curator/files/action/so-barracuda-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-barracuda:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-barracuda + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-beats-warm.yml b/salt/curator/files/action/so-beats-warm.yml new file mode 100644 index 000000000..da9f76656 --- /dev/null +++ b/salt/curator/files/action/so-beats-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-beats:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-beats + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-bluecoat-warm.yml b/salt/curator/files/action/so-bluecoat-warm.yml new file mode 100644 index 000000000..47a8d712f --- /dev/null +++ b/salt/curator/files/action/so-bluecoat-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-bluecoat:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-bluecoat + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-cef-warm.yml b/salt/curator/files/action/so-cef-warm.yml new file mode 100644 index 000000000..0a79fd2ba --- /dev/null +++ b/salt/curator/files/action/so-cef-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-cef:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-cef + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-checkpoint-warm.yml b/salt/curator/files/action/so-checkpoint-warm.yml new file mode 100644 index 000000000..0aaec1e19 --- /dev/null +++ b/salt/curator/files/action/so-checkpoint-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-checkpoint:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-checkpoint + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-cisco-warm.yml b/salt/curator/files/action/so-cisco-warm.yml new file mode 100644 index 000000000..a143a95c2 --- /dev/null +++ b/salt/curator/files/action/so-cisco-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-cisco:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-cisco + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-cyberark-warm.yml b/salt/curator/files/action/so-cyberark-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-cyberark-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-cylance-warm.yml b/salt/curator/files/action/so-cylance-warm.yml new file mode 100644 index 000000000..c9da7e68a --- /dev/null +++ b/salt/curator/files/action/so-cylance-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-cylance:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-cylance + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-elasticsearch-warm.yml b/salt/curator/files/action/so-elasticsearch-warm.yml new file mode 100644 index 000000000..9d82fc27b --- /dev/null +++ b/salt/curator/files/action/so-elasticsearch-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-elasticsearch:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-elasticsearch + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-f5-warm.yml b/salt/curator/files/action/so-f5-warm.yml new file mode 100644 index 000000000..12fbbe7ad --- /dev/null +++ b/salt/curator/files/action/so-f5-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-f5:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-f5 + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-firewall-warm.yml b/salt/curator/files/action/so-firewall-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-firewall-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-fortinet-warm.yml b/salt/curator/files/action/so-fortinet-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-fortinet-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-gcp-warm.yml b/salt/curator/files/action/so-gcp-warm.yml new file mode 100644 index 000000000..3bb9eee80 --- /dev/null +++ b/salt/curator/files/action/so-gcp-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-gcp:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-gcp + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-google_workspace.yml b/salt/curator/files/action/so-google_workspace-delete.yml similarity index 100% rename from salt/curator/files/action/so-google_workspace.yml rename to salt/curator/files/action/so-google_workspace-delete.yml diff --git a/salt/curator/files/action/so-google_workspace-warm.yml b/salt/curator/files/action/so-google_workspace-warm.yml new file mode 100644 index 000000000..7eb2d883f --- /dev/null +++ b/salt/curator/files/action/so-google_workspace-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-google_workspace:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-google_workspace + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-ids-warm.yml b/salt/curator/files/action/so-ids-warm.yml new file mode 100644 index 000000000..0edad5b5b --- /dev/null +++ b/salt/curator/files/action/so-ids-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-ids:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-ids + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-imperva-warm.yml b/salt/curator/files/action/so-imperva-warm.yml new file mode 100644 index 000000000..0297d5cd6 --- /dev/null +++ b/salt/curator/files/action/so-imperva-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-imperva:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-imperva + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-import-warm.yml b/salt/curator/files/action/so-import-warm.yml new file mode 100644 index 000000000..3a6fa3d3d --- /dev/null +++ b/salt/curator/files/action/so-import-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-import:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-import + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-infoblox-warm.yml b/salt/curator/files/action/so-infoblox-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-infoblox-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-juniper-warm.yml b/salt/curator/files/action/so-juniper-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-juniper-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-kibana-warm.yml b/salt/curator/files/action/so-kibana-warm.yml new file mode 100644 index 000000000..b5674c8c3 --- /dev/null +++ b/salt/curator/files/action/so-kibana-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-kibana:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-kibana + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-logstash-warm.yml b/salt/curator/files/action/so-logstash-warm.yml new file mode 100644 index 000000000..826bf2975 --- /dev/null +++ b/salt/curator/files/action/so-logstash-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-logstash:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-logstash + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-microsoft-warm.yml b/salt/curator/files/action/so-microsoft-warm.yml new file mode 100644 index 000000000..551d0cb56 --- /dev/null +++ b/salt/curator/files/action/so-microsoft-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-microsoft:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-microsoft + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-misp-warm.yml b/salt/curator/files/action/so-misp-warm.yml new file mode 100644 index 000000000..af29975b0 --- /dev/null +++ b/salt/curator/files/action/so-misp-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-misp:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-misp + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-netflow-warm.yml b/salt/curator/files/action/so-netflow-warm.yml new file mode 100644 index 000000000..ea57bb72b --- /dev/null +++ b/salt/curator/files/action/so-netflow-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-netflow:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-netflow + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-netscout-delete.yml b/salt/curator/files/action/so-netscout-delete.yml new file mode 100644 index 000000000..3c0e249b5 --- /dev/null +++ b/salt/curator/files/action/so-netscout-delete.yml @@ -0,0 +1,29 @@ +{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-netscout:delete', 365) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: delete_indices + description: >- + Delete netscout indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-netscout.*|so-netscout.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + \ No newline at end of file diff --git a/salt/curator/files/action/so-netscout-warm.yml b/salt/curator/files/action/so-netscout-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-netscout-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-okta-warm.yml b/salt/curator/files/action/so-okta-warm.yml new file mode 100644 index 000000000..75764860d --- /dev/null +++ b/salt/curator/files/action/so-okta-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-okta:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-okta + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-osquery-warm.yml b/salt/curator/files/action/so-osquery-warm.yml new file mode 100644 index 000000000..156a83c7a --- /dev/null +++ b/salt/curator/files/action/so-osquery-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-osquery:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-osquery + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-ossec-warm.yml b/salt/curator/files/action/so-ossec-warm.yml new file mode 100644 index 000000000..769d6cbea --- /dev/null +++ b/salt/curator/files/action/so-ossec-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-ossec:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-ossec + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-proofpoint-warm.yml b/salt/curator/files/action/so-proofpoint-warm.yml new file mode 100644 index 000000000..8304ae41a --- /dev/null +++ b/salt/curator/files/action/so-proofpoint-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-proofpoint:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-proofpoint + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-radware-warm.yml b/salt/curator/files/action/so-radware-warm.yml new file mode 100644 index 000000000..8d4337aaf --- /dev/null +++ b/salt/curator/files/action/so-radware-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-radware:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-radware + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-redis-warm.yml b/salt/curator/files/action/so-redis-warm.yml new file mode 100644 index 000000000..a5b1055c3 --- /dev/null +++ b/salt/curator/files/action/so-redis-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-redis:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-redis + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-snort-warm.yml b/salt/curator/files/action/so-snort-warm.yml new file mode 100644 index 000000000..3bbc977e2 --- /dev/null +++ b/salt/curator/files/action/so-snort-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-snort:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-snort + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-snyk-warm.yml b/salt/curator/files/action/so-snyk-warm.yml new file mode 100644 index 000000000..f9b10bbdd --- /dev/null +++ b/salt/curator/files/action/so-snyk-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-snyk:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-snyk + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-sonicwall-warm.yml b/salt/curator/files/action/so-sonicwall-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-sonicwall-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-sophos-warm.yml b/salt/curator/files/action/so-sophos-warm.yml new file mode 100644 index 000000000..40cc60084 --- /dev/null +++ b/salt/curator/files/action/so-sophos-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-sophos:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-sophos + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-strelka-warm.yml b/salt/curator/files/action/so-strelka-warm.yml new file mode 100644 index 000000000..cfa88b0c1 --- /dev/null +++ b/salt/curator/files/action/so-strelka-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-strelka:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-strelka + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-syslog-warm.yml b/salt/curator/files/action/so-syslog-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-syslog-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-tomcat-warm.yml b/salt/curator/files/action/so-tomcat-warm.yml new file mode 100644 index 000000000..8fb7884c0 --- /dev/null +++ b/salt/curator/files/action/so-tomcat-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-tomcat:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-tomcat + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-zeek-warm.yml b/salt/curator/files/action/so-zeek-warm.yml new file mode 100644 index 000000000..5369ed9a9 --- /dev/null +++ b/salt/curator/files/action/so-zeek-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-aws + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/so-zscaler-warm.yml b/salt/curator/files/action/so-zscaler-warm.yml new file mode 100644 index 000000000..8a7d8187a --- /dev/null +++ b/salt/curator/files/action/so-zscaler-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-zscaler:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-zscaler + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + diff --git a/salt/curator/files/action/warm.yml b/salt/curator/files/action/warm.yml deleted file mode 100644 index efa2196b2..000000000 --- a/salt/curator/files/action/warm.yml +++ /dev/null @@ -1,23 +0,0 @@ -#actions: -# 1: -# action: allocation -# description: "Apply shard allocation filtering rules to the specified indices" -# options: -# key: box_type -# value: warm -# allocation_type: require -# wait_for_completion: true -# timeout_override: -# continue_if_exception: false -# disable_action: false -# filters: -# - filtertype: pattern -# kind: prefix -# value: so- -# - filtertype: age -# source: name -# direction: older -# timestring: '%Y.%m.%d' -# unit: days -# unit_count: 3 - diff --git a/salt/curator/files/bin/so-cluster-warm b/salt/curator/files/bin/so-cluster-warm new file mode 100644 index 000000000..c8234c7c9 --- /dev/null +++ b/salt/curator/files/bin/so-cluster-warm @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +APP=warm +lf=/tmp/$APP-pidLockFile +# create empty lock file if none exists +cat /dev/null >> $lf +read lastPID < $lf +# if lastPID is not null and a process with that pid exists , exit +[ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit +echo $$ > $lf + +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/warm.yml > /dev/null 2>&1 \ No newline at end of file From ed116cf850bcbfaffcfd4b1aa04e96828781c5ad Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 15:09:26 -0400 Subject: [PATCH 31/51] Add Actions for warm indices --- salt/curator/files/action/so-cyberark-warm.yml | 4 ++-- salt/curator/files/action/so-firewall-warm.yml | 4 ++-- salt/curator/files/action/so-fortinet-warm.yml | 4 ++-- salt/curator/files/action/so-netscout-warm.yml | 4 ++-- salt/curator/files/action/so-sonicwall-warm.yml | 4 ++-- salt/curator/files/action/so-syslog-warm.yml | 4 ++-- salt/curator/files/action/so-zeek-warm.yml | 4 ++-- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/salt/curator/files/action/so-cyberark-warm.yml b/salt/curator/files/action/so-cyberark-warm.yml index 5369ed9a9..8eae0b542 100644 --- a/salt/curator/files/action/so-cyberark-warm.yml +++ b/salt/curator/files/action/so-cyberark-warm.yml @@ -1,4 +1,4 @@ -{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-cyberark:warm', 7) -%} actions: 1: action: allocation @@ -14,7 +14,7 @@ actions: filters: - filtertype: pattern kind: prefix - value: so-aws + value: so-cyberark - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-firewall-warm.yml b/salt/curator/files/action/so-firewall-warm.yml index 5369ed9a9..2e9643dc3 100644 --- a/salt/curator/files/action/so-firewall-warm.yml +++ b/salt/curator/files/action/so-firewall-warm.yml @@ -1,4 +1,4 @@ -{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-firewall:warm', 7) -%} actions: 1: action: allocation @@ -14,7 +14,7 @@ actions: filters: - filtertype: pattern kind: prefix - value: so-aws + value: so-firewall - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-fortinet-warm.yml b/salt/curator/files/action/so-fortinet-warm.yml index 5369ed9a9..db9a6f2db 100644 --- a/salt/curator/files/action/so-fortinet-warm.yml +++ b/salt/curator/files/action/so-fortinet-warm.yml @@ -1,4 +1,4 @@ -{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-fortinet:warm', 7) -%} actions: 1: action: allocation @@ -14,7 +14,7 @@ actions: filters: - filtertype: pattern kind: prefix - value: so-aws + value: so-fortinet - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-netscout-warm.yml b/salt/curator/files/action/so-netscout-warm.yml index 5369ed9a9..1b93c3118 100644 --- a/salt/curator/files/action/so-netscout-warm.yml +++ b/salt/curator/files/action/so-netscout-warm.yml @@ -1,4 +1,4 @@ -{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-netscout:warm', 7) -%} actions: 1: action: allocation @@ -14,7 +14,7 @@ actions: filters: - filtertype: pattern kind: prefix - value: so-aws + value: so-netscout - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-sonicwall-warm.yml b/salt/curator/files/action/so-sonicwall-warm.yml index 5369ed9a9..fa8ceb3e4 100644 --- a/salt/curator/files/action/so-sonicwall-warm.yml +++ b/salt/curator/files/action/so-sonicwall-warm.yml @@ -1,4 +1,4 @@ -{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-sonicwall:warm', 7) -%} actions: 1: action: allocation @@ -14,7 +14,7 @@ actions: filters: - filtertype: pattern kind: prefix - value: so-aws + value: so-sonicwall - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-syslog-warm.yml b/salt/curator/files/action/so-syslog-warm.yml index 5369ed9a9..e5ebb2fa6 100644 --- a/salt/curator/files/action/so-syslog-warm.yml +++ b/salt/curator/files/action/so-syslog-warm.yml @@ -1,4 +1,4 @@ -{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-syslog:warm', 7) -%} actions: 1: action: allocation @@ -14,7 +14,7 @@ actions: filters: - filtertype: pattern kind: prefix - value: so-aws + value: so-syslog - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-zeek-warm.yml b/salt/curator/files/action/so-zeek-warm.yml index 5369ed9a9..2b4b6a729 100644 --- a/salt/curator/files/action/so-zeek-warm.yml +++ b/salt/curator/files/action/so-zeek-warm.yml @@ -1,4 +1,4 @@ -{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:warm', 7) -%} actions: 1: action: allocation @@ -14,7 +14,7 @@ actions: filters: - filtertype: pattern kind: prefix - value: so-aws + value: so-zeek - filtertype: age source: name direction: older From f6f4375e1396956124d875388e6176a8fb2873da Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 15:34:26 -0400 Subject: [PATCH 32/51] Add logic for cronjobs --- ...o-cluster-warm => so-curator-cluster-warm} | 0 salt/curator/init.sls | 127 ++++++++++++------ 2 files changed, 85 insertions(+), 42 deletions(-) rename salt/curator/files/bin/{so-cluster-warm => so-curator-cluster-warm} (100%) diff --git a/salt/curator/files/bin/so-cluster-warm b/salt/curator/files/bin/so-curator-cluster-warm similarity index 100% rename from salt/curator/files/bin/so-cluster-warm rename to salt/curator/files/bin/so-curator-cluster-warm diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 1acea5d25..57da8bfa1 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -5,6 +5,9 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} {% set REMOVECURATORCRON = False %} +{% set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %} +{% set HOTWARM = salt['pillar.get']('elasticsearch:hot_warm_enabled', False) %} + {% if grains['role'] in ['so-eval', 'so-node', 'so-managersearch', 'so-heavynode', 'so-standalone', 'so-manager'] %} {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} {% from "curator/map.jinja" import CURATOROPTIONS with context %} @@ -148,62 +151,102 @@ delete_so-curator_so-status: {% if REMOVECURATORCRON %} so-curatorcloseddeletecron: - cron.absent: - - name: /usr/sbin/so-curator-closed-delete > /opt/so/log/curator/cron-closed-delete.log 2>&1 - - user: root + cron.absent: + - name: /usr/sbin/so-curator-closed-delete > /opt/so/log/curator/cron-closed-delete.log 2>&1 + - user: root so-curatorclosecron: - cron.absent: - - name: /usr/sbin/so-curator-close > /opt/so/log/curator/cron-close.log 2>&1 - - user: root + cron.absent: + - name: /usr/sbin/so-curator-close > /opt/so/log/curator/cron-close.log 2>&1 + - user: root so-curatordeletecron: - cron.absent: - - name: /usr/sbin/so-curator-delete > /opt/so/log/curator/cron-delete.log 2>&1 - - user: root + cron.absent: + - name: /usr/sbin/so-curator-delete > /opt/so/log/curator/cron-delete.log 2>&1 + - user: root {% else %} + {% if TRUECLUSTER is sameas true %} so-curatorcloseddeletecron: - cron.present: - - name: /usr/sbin/so-curator-closed-delete > /opt/so/log/curator/cron-closed-delete.log 2>&1 - - user: root - - minute: '*' - - hour: '*' - - daymonth: '*' - - month: '*' - - dayweek: '*' + cron.absent: + - name: /usr/sbin/so-curator-closed-delete > /opt/so/log/curator/cron-closed-delete.log 2>&1 + - user: root so-curatorclosecron: - cron.present: - - name: /usr/sbin/so-curator-close > /opt/so/log/curator/cron-close.log 2>&1 - - user: root - - minute: '*' - - hour: '*' - - daymonth: '*' - - month: '*' - - dayweek: '*' + cron.absent: + - name: /usr/sbin/so-curator-close > /opt/so/log/curator/cron-close.log 2>&1 + - user: root so-curatordeletecron: - cron.present: - - name: /usr/sbin/so-curator-delete > /opt/so/log/curator/cron-delete.log 2>&1 - - user: root - - minute: '*' - - hour: '*' - - daymonth: '*' - - month: '*' - - dayweek: '*' + cron.absent: + - name: /usr/sbin/so-curator-delete > /opt/so/log/curator/cron-delete.log 2>&1 + - user: root + +so-curatorclusterclose: + cron.present: + - name: /usr/sbin/so-curator-cluster-close > /opt/so/log/curator/cron-close.log 2>&1 + - user: root + - minute: '*' + - hour: '*/1' + - daymonth: '*' + - month: '*' + - dayweek: '*' +so-curatorclusterdelete: + cron.present: + - name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-close.log 2>&1 + - user: root + - minute: '*' + - hour: '*/1' + - daymonth: '*' + - month: '*' + - dayweek: '*' + {% if HOTWARM is sameas true %} +so-curatorclusterwarm: + cron.present: + - name: /usr/sbin/so-curator-cluster-warm > /opt/so/log/curator/cron-close.log 2>&1 + - user: root + - minute: '*' + - hour: '*/1' + - daymonth: '*' + - month: '*' + - dayweek: '*' + {% endif %%} + + {% else %} +so-curatorcloseddeletecron: + cron.present: + - name: /usr/sbin/so-curator-closed-delete > /opt/so/log/curator/cron-closed-delete.log 2>&1 + - user: root + - minute: '*/5' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' + +so-curatorclosecron: + cron.present: + - name: /usr/sbin/so-curator-close > /opt/so/log/curator/cron-close.log 2>&1 + - user: root + - minute: '*/5' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' + +so-curatordeletecron: + cron.present: + - name: /usr/sbin/so-curator-delete > /opt/so/log/curator/cron-delete.log 2>&1 + - user: root + - minute: '*/5' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' + + {% endif %} {% endif %} - -# Begin Curator Cron Jobs - -# Close -# Delete -# Hot Warm -# Segment Merge - -# End Curator Cron Jobs {% endif %} {% else %} From 9f51f02ab48538a014b866012991da26ed953e4f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 15:40:09 -0400 Subject: [PATCH 33/51] Add logic for cronjobs --- salt/curator/init.sls | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 57da8bfa1..8900c35d8 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -187,7 +187,7 @@ so-curatorclusterclose: cron.present: - name: /usr/sbin/so-curator-cluster-close > /opt/so/log/curator/cron-close.log 2>&1 - user: root - - minute: '*' + - minute: '2' - hour: '*/1' - daymonth: '*' - month: '*' @@ -197,7 +197,7 @@ so-curatorclusterdelete: cron.present: - name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-close.log 2>&1 - user: root - - minute: '*' + - minute: '2' - hour: '*/1' - daymonth: '*' - month: '*' @@ -207,7 +207,7 @@ so-curatorclusterwarm: cron.present: - name: /usr/sbin/so-curator-cluster-warm > /opt/so/log/curator/cron-close.log 2>&1 - user: root - - minute: '*' + - minute: '2' - hour: '*/1' - daymonth: '*' - month: '*' From dc25ed2594b01430ca23722b6bcd9deb1c01043a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 15:43:48 -0400 Subject: [PATCH 34/51] Add logic for cronjobs --- salt/curator/init.sls | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 8900c35d8..b7b580af7 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -167,22 +167,7 @@ so-curatordeletecron: {% else %} - {% if TRUECLUSTER is sameas true %} -so-curatorcloseddeletecron: - cron.absent: - - name: /usr/sbin/so-curator-closed-delete > /opt/so/log/curator/cron-closed-delete.log 2>&1 - - user: root - -so-curatorclosecron: - cron.absent: - - name: /usr/sbin/so-curator-close > /opt/so/log/curator/cron-close.log 2>&1 - - user: root - -so-curatordeletecron: - cron.absent: - - name: /usr/sbin/so-curator-delete > /opt/so/log/curator/cron-delete.log 2>&1 - - user: root - + {% if TRUECLUSTER is sameas true %} so-curatorclusterclose: cron.present: - name: /usr/sbin/so-curator-cluster-close > /opt/so/log/curator/cron-close.log 2>&1 From c12f380bc348c10b4b1adda9d65f22964f29539f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 15:51:34 -0400 Subject: [PATCH 35/51] Only enable curator on Manager in true cluster --- salt/curator/map.jinja | 4 ++-- salt/top.sls | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/curator/map.jinja b/salt/curator/map.jinja index 470143a4f..bbcff2a9c 100644 --- a/salt/curator/map.jinja +++ b/salt/curator/map.jinja @@ -4,10 +4,10 @@ {% do CURATOROPTIONS.update({'manage_sostatus': True}) %} # don't start the docker container if curator is disabled via pillar -{% if not ENABLED or grains.id.split('_')|last == 'manager'%} +{% if not ENABLED or (TRUECLUSTER and grains.id.split('_')|last == 'searchnode') or (not TRUECLUSTER and grains.id.split('_')|last == 'manager') %} {% do CURATOROPTIONS.update({'start': False}) %} {% do CURATOROPTIONS.update({'status': 'absent'}) %} - {% if grains.id.split('_')|last == 'manager' %} + {% if (TRUECLUSTER and grains.id.split('_')|last == 'searchnode') or (not TRUECLUSTER and grains.id.split('_')|last == 'manager') %} {% do CURATOROPTIONS.update({'manage_sostatus': False}) %} {% endif %} {% else %} diff --git a/salt/top.sls b/salt/top.sls index b38d315d8..e2c980e1d 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -189,6 +189,7 @@ base: {%- if KIBANA %} - kibana {%- endif %} + - curator {%- if ELASTALERT %} - elastalert {%- endif %} From 5b9ef3bc0d2ed5b8881398ebe89a45b7019e9fd2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 15:55:44 -0400 Subject: [PATCH 36/51] Only enable curator on Manager in true cluster --- salt/curator/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/curator/init.sls b/salt/curator/init.sls index b7b580af7..10cffe680 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -197,7 +197,7 @@ so-curatorclusterwarm: - daymonth: '*' - month: '*' - dayweek: '*' - {% endif %%} + {% endif %} {% else %} so-curatorcloseddeletecron: From 0c8e11dc9f3903b8cd51e26642d217aa15bd7eb9 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 16:13:05 -0400 Subject: [PATCH 37/51] Only enable curator on Manager in true cluster --- salt/curator/init.sls | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 10cffe680..95e62ac95 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -92,6 +92,30 @@ curdel: - group: 939 - mode: 755 +curclusterclose: + file.managed: + - name: /usr/sbin/so-curator-cluster-close + - source: salt://curator/files/bin/so-curator-cluster-close + - user: 934 + - group: 939 + - mode: 755 + +curclusterdelete: + file.managed: + - name: /usr/sbin/so-curator-cluster-delete + - source: salt://curator/files/bin/so-curator-cluster-delete + - user: 934 + - group: 939 + - mode: 755 + +curclustercwarm: + file.managed: + - name: /usr/sbin/so-curator-cluster-warm + - source: salt://curator/files/bin/so-curator-cluster-warm + - user: 934 + - group: 939 + - mode: 755 + so-curator: docker_container.{{ CURATOROPTIONS.status }}: {% if CURATOROPTIONS.status == 'running' %} From 958575c22a2fdbe4f09910c65960305fa1561b73 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 16:17:55 -0400 Subject: [PATCH 38/51] Only enable curator on Manager in true cluster --- salt/curator/init.sls | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 95e62ac95..54fcff3a6 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -99,6 +99,7 @@ curclusterclose: - user: 934 - group: 939 - mode: 755 + - template: jinja curclusterdelete: file.managed: @@ -107,6 +108,7 @@ curclusterdelete: - user: 934 - group: 939 - mode: 755 + - template: jinja curclustercwarm: file.managed: @@ -115,7 +117,8 @@ curclustercwarm: - user: 934 - group: 939 - mode: 755 - + - template: jinja + so-curator: docker_container.{{ CURATOROPTIONS.status }}: {% if CURATOROPTIONS.status == 'running' %} From 62edfd0b7fde61491cf75c5ec499e1ca5ada16ec Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 16:20:42 -0400 Subject: [PATCH 39/51] Only enable curator on Manager in true cluster --- salt/curator/files/bin/so-curator-cluster-close | 4 ++-- salt/curator/files/bin/so-curator-cluster-delete | 4 ++-- salt/curator/files/bin/so-curator-cluster-warm | 16 +++++++++++++++- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/salt/curator/files/bin/so-curator-cluster-close b/salt/curator/files/bin/so-curator-cluster-close index 1a4124b53..b2043d3b6 100644 --- a/salt/curator/files/bin/so-curator-cluster-close +++ b/salt/curator/files/bin/so-curator-cluster-close @@ -38,7 +38,7 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-close.yml > /dev/null 2>&1; {% for INDEX in THIRDPARTY -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1; -{% done -%} +{% endfor -%} {% for INDEX in SO -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} -{% done -%} \ No newline at end of file +{% endfor -%} \ No newline at end of file diff --git a/salt/curator/files/bin/so-curator-cluster-delete b/salt/curator/files/bin/so-curator-cluster-delete index cccb270b0..df7f2c4bf 100644 --- a/salt/curator/files/bin/so-curator-cluster-delete +++ b/salt/curator/files/bin/so-curator-cluster-delete @@ -38,7 +38,7 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-delete.yml > /dev/null 2>&1; {% for INDEX in THIRDPARTY -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-delete.yml > /dev/null 2>&1; -{% done -%} +{% endfor -%} {% for INDEX in SO -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-delete.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} -{% done -%} \ No newline at end of file +{% endfor -%} \ No newline at end of file diff --git a/salt/curator/files/bin/so-curator-cluster-warm b/salt/curator/files/bin/so-curator-cluster-warm index c8234c7c9..db54b7fe9 100644 --- a/salt/curator/files/bin/so-curator-cluster-warm +++ b/salt/curator/files/bin/so-curator-cluster-warm @@ -24,4 +24,18 @@ read lastPID < $lf [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit echo $$ > $lf -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/warm.yml > /dev/null 2>&1 \ No newline at end of file +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-warm.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-warm.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-firewall-warm.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ids-warm.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-import-warm.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-osquery-warm.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-warm.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-warm.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-warm.yml > /dev/null 2>&1; +{% for INDEX in THIRDPARTY -%} +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-warm.yml > /dev/null 2>&1; +{% endfor -%} +{% for INDEX in SO -%} +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-warm.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} +{% endfor -%} \ No newline at end of file From e7b04b862fb2eb21b358fa6e20d682db6d0404aa Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 16:21:48 -0400 Subject: [PATCH 40/51] Only enable curator on Manager in true cluster --- salt/curator/files/bin/so-curator-cluster-warm | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/curator/files/bin/so-curator-cluster-warm b/salt/curator/files/bin/so-curator-cluster-warm index db54b7fe9..ec08a31c0 100644 --- a/salt/curator/files/bin/so-curator-cluster-warm +++ b/salt/curator/files/bin/so-curator-cluster-warm @@ -24,6 +24,9 @@ read lastPID < $lf [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit echo $$ > $lf +{% from 'filebeat/map.jinja' import THIRDPARTY with context %} +{% from 'filebeat/map.jinja' import SO with context %} + docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-warm.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-warm.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-firewall-warm.yml > /dev/null 2>&1; From e09aa4e5d4b359cd7885f58df003429bae495912 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 16:35:19 -0400 Subject: [PATCH 41/51] Only enable curator on Manager in true cluster --- salt/curator/files/bin/so-curator-cluster-close | 4 ++-- salt/curator/files/bin/so-curator-cluster-delete | 4 ++-- salt/curator/files/bin/so-curator-cluster-warm | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/curator/files/bin/so-curator-cluster-close b/salt/curator/files/bin/so-curator-cluster-close index b2043d3b6..7b8c41c9e 100644 --- a/salt/curator/files/bin/so-curator-cluster-close +++ b/salt/curator/files/bin/so-curator-cluster-close @@ -36,9 +36,9 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-close.yml > /dev/null 2>&1; -{% for INDEX in THIRDPARTY -%} +{% for INDEX in THIRDPARTY.modules.keys() -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1; {% endfor -%} -{% for INDEX in SO -%} +{% for INDEX in SO.modules.keys() -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} {% endfor -%} \ No newline at end of file diff --git a/salt/curator/files/bin/so-curator-cluster-delete b/salt/curator/files/bin/so-curator-cluster-delete index df7f2c4bf..21195da5d 100644 --- a/salt/curator/files/bin/so-curator-cluster-delete +++ b/salt/curator/files/bin/so-curator-cluster-delete @@ -36,9 +36,9 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-delete.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-delete.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-delete.yml > /dev/null 2>&1; -{% for INDEX in THIRDPARTY -%} +{% for INDEX in THIRDPARTY.modules.keys() -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-delete.yml > /dev/null 2>&1; {% endfor -%} -{% for INDEX in SO -%} +{% for INDEX in SO.modules.keys() -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-delete.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} {% endfor -%} \ No newline at end of file diff --git a/salt/curator/files/bin/so-curator-cluster-warm b/salt/curator/files/bin/so-curator-cluster-warm index ec08a31c0..4348ba2e3 100644 --- a/salt/curator/files/bin/so-curator-cluster-warm +++ b/salt/curator/files/bin/so-curator-cluster-warm @@ -36,9 +36,9 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-warm.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-warm.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-warm.yml > /dev/null 2>&1; -{% for INDEX in THIRDPARTY -%} +{% for INDEX in THIRDPARTY.modules.keys() -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-warm.yml > /dev/null 2>&1; {% endfor -%} -{% for INDEX in SO -%} +{% for INDEX in SO.modules.keys() -%} docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-warm.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} {% endfor -%} \ No newline at end of file From 61ebedc0e9439c43c57c4ae8f2fff118984f16e6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 16:56:08 -0400 Subject: [PATCH 42/51] Only enable curator on Manager in true cluster --- salt/curator/files/action/so-infoblox-warm.yml | 4 ++-- salt/curator/files/action/so-kibana-close.yml | 6 +++--- salt/curator/init.sls | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/salt/curator/files/action/so-infoblox-warm.yml b/salt/curator/files/action/so-infoblox-warm.yml index 5369ed9a9..a2f571b7a 100644 --- a/salt/curator/files/action/so-infoblox-warm.yml +++ b/salt/curator/files/action/so-infoblox-warm.yml @@ -1,4 +1,4 @@ -{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-aws:warm', 7) -%} +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-infoblox:warm', 7) -%} actions: 1: action: allocation @@ -14,7 +14,7 @@ actions: filters: - filtertype: pattern kind: prefix - value: so-aws + value: so-infoblox - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-kibana-close.yml b/salt/curator/files/action/so-kibana-close.yml index 05583d853..7347fb01c 100644 --- a/salt/curator/files/action/so-kibana-close.yml +++ b/salt/curator/files/action/so-kibana-close.yml @@ -1,4 +1,4 @@ -{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-ids:close', 30) -%} +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-kibana:close', 30) -%} --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" @@ -10,7 +10,7 @@ actions: 1: action: close description: >- - Close IDS indices older than {{cur_close_days}} days. + Close kibana indices older than {{cur_close_days}} days. options: delete_aliases: False timeout_override: @@ -19,7 +19,7 @@ actions: filters: - filtertype: pattern kind: regex - value: '^(logstash-ids.*|so-ids.*)$' + value: '^(logstash-kibana.*|so-kibana.*)$' - filtertype: age source: name direction: older diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 54fcff3a6..87e1fec33 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -207,7 +207,7 @@ so-curatorclusterclose: so-curatorclusterdelete: cron.present: - - name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-close.log 2>&1 + - name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-delete.log 2>&1 - user: root - minute: '2' - hour: '*/1' @@ -217,7 +217,7 @@ so-curatorclusterdelete: {% if HOTWARM is sameas true %} so-curatorclusterwarm: cron.present: - - name: /usr/sbin/so-curator-cluster-warm > /opt/so/log/curator/cron-close.log 2>&1 + - name: /usr/sbin/so-curator-cluster-warm > /opt/so/log/curator/cron-warm.log 2>&1 - user: root - minute: '2' - hour: '*/1' From 527a793e9419ada93a38257e90cf51b97ea90a16 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 31 Aug 2021 16:59:41 -0400 Subject: [PATCH 43/51] Only enable curator on Manager in true cluster --- salt/curator/files/action/so-o365-warm.yml | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 salt/curator/files/action/so-o365-warm.yml diff --git a/salt/curator/files/action/so-o365-warm.yml b/salt/curator/files/action/so-o365-warm.yml new file mode 100644 index 000000000..cbb7bc24e --- /dev/null +++ b/salt/curator/files/action/so-o365-warm.yml @@ -0,0 +1,24 @@ +{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-o365:warm', 7) -%} +actions: + 1: + action: allocation + description: "Apply shard allocation filtering rules to the specified indices" + options: + key: box_type + value: warm + allocation_type: require + wait_for_completion: true + timeout_override: + continue_if_exception: false + disable_action: false + filters: + - filtertype: pattern + kind: prefix + value: so-o365 + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ WARM_DAYS }} + From 007b15979a11a2297bf0badec2fae50c2ac1e7c6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 1 Sep 2021 09:25:14 -0400 Subject: [PATCH 44/51] Non Cluster honor closed indices values --- salt/curator/files/bin/so-curator-close | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/salt/curator/files/bin/so-curator-close b/salt/curator/files/bin/so-curator-close index 6f7b96790..c1e940b4d 100644 --- a/salt/curator/files/bin/so-curator-close +++ b/salt/curator/files/bin/so-curator-close @@ -23,5 +23,22 @@ read lastPID < $lf # if lastPID is not null and a process with that pid exists , exit [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit echo $$ > $lf +{% from 'filebeat/map.jinja' import THIRDPARTY with context %} +{% from 'filebeat/map.jinja' import SO with context %} -/usr/sbin/so-curator-closed-delete > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-firewall-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ids-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-import-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-osquery-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-close.yml > /dev/null 2>&1 +/usr/sbin/so-curator-closed-delete > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-firewall-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ids-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-import-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-osquery-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-close.yml > /dev/null 2>&1; +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-close.yml > /dev/null 2>&1; +{% for INDEX in THIRDPARTY.modules.keys() -%} +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1; +{% endfor -%} +{% for INDEX in SO.modules.keys() -%} +docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} +{% endfor -%} From 324e6b12e2bbad13686c172e09559542de810231 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 1 Sep 2021 09:32:32 -0400 Subject: [PATCH 45/51] Add jinja template --- salt/curator/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 87e1fec33..c604571ba 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -83,6 +83,7 @@ curclose: - user: 934 - group: 939 - mode: 755 + - template: jinja curdel: file.managed: From 576c893eb3731817ecf9eb709467e6d6ab640f3a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 1 Sep 2021 15:08:53 -0400 Subject: [PATCH 46/51] Exit on missing file argument --- salt/common/tools/sbin/soup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 60e0ce93f..87fe56784 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -788,6 +788,7 @@ main() { check_airgap if [[ $is_airgap -eq 0 && $UNATTENDED == true && -z $ISOLOC ]]; then echo "Missing file argument for unattended airgap upgrade." + exit 1 fi echo "Checking to see if this is a manager." echo "" From 446821e9fdd657d580787e39835b2d55a368aa58 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 1 Sep 2021 15:11:18 -0400 Subject: [PATCH 47/51] Use exit code 0 when printing error message before exiting soup --- salt/common/tools/sbin/soup | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 87fe56784..de26e73ea 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -127,7 +127,7 @@ EOF if [ ! -f /tmp/soagupdate/SecurityOnion/VERSION ]; then echo "Something went wrong trying to mount the ISO." echo "Ensure you verify the ISO that you downloaded." - exit 1 + exit 0 else echo "ISO has been mounted!" fi @@ -140,14 +140,14 @@ EOF if [ ! -f /tmp/soagupdate/SecurityOnion/VERSION ]; then echo "Something went wrong trying to mount the device." echo "Ensure you verify the ISO that you downloaded." - exit 1 + exit 0 else echo "Device has been mounted!" fi else echo "Could not find Security Onion ISO content at ${ISOLOC}" echo "Ensure the path you entered is correct, and that you verify the ISO that you downloaded." - exit 1 + exit 0 fi fi } @@ -157,7 +157,7 @@ airgap_update_dockers() { # Let's copy the tarball if [[ ! -f $AGDOCKER/registry.tar ]]; then echo "Unable to locate registry. Exiting" - exit 1 + exit 0 else echo "Stopping the registry docker" docker stop so-dockerregistry @@ -631,7 +631,7 @@ upgrade_space() { clean_dockers if ! verify_upgradespace; then echo "There is not enough space to perform the upgrade. Please free up space and try again" - exit 1 + exit 0 fi else echo "You have enough space for upgrade. Proceeding with soup." @@ -788,7 +788,7 @@ main() { check_airgap if [[ $is_airgap -eq 0 && $UNATTENDED == true && -z $ISOLOC ]]; then echo "Missing file argument for unattended airgap upgrade." - exit 1 + exit 0 fi echo "Checking to see if this is a manager." echo "" @@ -885,7 +885,7 @@ main() { echo "Once the issue is resolved, run soup again." echo "Exiting." echo "" - exit 1 + exit 0 else echo "Salt upgrade success." echo "" From 556bad6925f59cb98bba37f88d46bda080856b2d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 1 Sep 2021 15:13:43 -0400 Subject: [PATCH 48/51] Add eventfields for new default logs --- salt/soc/files/soc/hunt.eventfields.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/hunt.eventfields.json b/salt/soc/files/soc/hunt.eventfields.json index e8af03a5c..354aa31ce 100644 --- a/salt/soc/files/soc/hunt.eventfields.json +++ b/salt/soc/files/soc/hunt.eventfields.json @@ -40,5 +40,9 @@ ":strelka:file": ["soc_timestamp", "file.name", "file.size", "hash.md5", "file.source", "file.mime_type", "log.id.fuid" ], ":suricata:": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "rule.name", "rule.category", "event.severity_label", "log.id.uid", "network.community_id" ], ":sysmon:": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "source.hostname", "event.dataset", "process.executable", "user.name" ], - ":windows_eventlog:": ["soc_timestamp", "user.name" ] + ":windows_eventlog:": ["soc_timestamp", "user.name" ], + ":elasticsearch:": ["soc_timestamp", "agent.name", "message", "log.level", "metadata.version", "metadata.pipeline", "event.dataset" ], + ":kibana:": ["soc_timestamp", "host.name", "message", "kibana.log.meta.req.headers.x-real-ip", "event.dataset" ], + "::rootcheck": ["soc_timestamp", "host.name", "metadata.ip_address", "log.full", "event.dataset", "event.module" ], + "::syscollector": ["soc_timestamp", "host.name", "metadata.ip_address", "wazuh.data.type", "event.dataset", "event.module" ] } From 0ad3d826ebf7ea3cccc08c8a392a5e064b1bde1f Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 2 Sep 2021 07:07:36 -0400 Subject: [PATCH 49/51] Invalid password message should also mention that dollar signs are not allowed --- salt/common/tools/sbin/so-fleet-user-add | 2 +- salt/common/tools/sbin/so-fleet-user-update | 2 +- salt/common/tools/sbin/so-thehive-user-add | 2 +- salt/common/tools/sbin/so-thehive-user-update | 2 +- setup/so-whiptail | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/so-fleet-user-add b/salt/common/tools/sbin/so-fleet-user-add index e905424a9..9d80c2076 100755 --- a/salt/common/tools/sbin/so-fleet-user-add +++ b/salt/common/tools/sbin/so-fleet-user-add @@ -42,7 +42,7 @@ fi read -rs FLEET_PASS if ! check_password "$FLEET_PASS"; then - echo "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password." + echo "Password is invalid. Please exclude single quotes, double quotes, dollar signs, and backslashes from the password." exit 2 fi diff --git a/salt/common/tools/sbin/so-fleet-user-update b/salt/common/tools/sbin/so-fleet-user-update index e6a142d1d..793f7b622 100755 --- a/salt/common/tools/sbin/so-fleet-user-update +++ b/salt/common/tools/sbin/so-fleet-user-update @@ -52,7 +52,7 @@ fi read -rs FLEET_PASS if ! check_password "$FLEET_PASS"; then - echo "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password." + echo "Password is invalid. Please exclude single quotes, double quotes, dollar signs, and backslashes from the password." exit 2 fi diff --git a/salt/common/tools/sbin/so-thehive-user-add b/salt/common/tools/sbin/so-thehive-user-add index 5d174fdf1..e26dc58fc 100755 --- a/salt/common/tools/sbin/so-thehive-user-add +++ b/salt/common/tools/sbin/so-thehive-user-add @@ -42,7 +42,7 @@ fi read -rs THEHIVE_PASS if ! check_password "$THEHIVE_PASS"; then - echo "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password." + echo "Password is invalid. Please exclude single quotes, double quotes, dollar signs, and backslashes from the password." exit 2 fi diff --git a/salt/common/tools/sbin/so-thehive-user-update b/salt/common/tools/sbin/so-thehive-user-update index 6df199f6a..fdda5eaa7 100755 --- a/salt/common/tools/sbin/so-thehive-user-update +++ b/salt/common/tools/sbin/so-thehive-user-update @@ -42,7 +42,7 @@ fi read -rs THEHIVE_PASS if ! check_password "$THEHIVE_PASS"; then - echo "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password." + echo "Password is invalid. Please exclude single quotes, double quotes, dollar signs, and backslashes from the password." exit 2 fi diff --git a/setup/so-whiptail b/setup/so-whiptail index 780411841..125335729 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -843,7 +843,7 @@ whiptail_invalid_pass_characters_warning() { [ -n "$TESTING" ] && return - whiptail --title "$whiptail_title" --msgbox "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password." 8 75 + whiptail --title "$whiptail_title" --msgbox "Password is invalid. Please exclude single quotes, double quotes, dollar signs, and backslashes from the password." 8 75 } whiptail_invalid_pass_warning() { From 56894b95813a4566ca18ab035882c513961ad5e4 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 2 Sep 2021 11:15:32 -0400 Subject: [PATCH 50/51] Add logic to check unattended flag when checking if updates are available --- salt/common/tools/sbin/soup | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index de26e73ea..17339fdd5 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -289,25 +289,31 @@ check_os_updates() { OSUPDATES=$(yum -q list updates | wc -l) fi if [[ "$OSUPDATES" -gt 0 ]]; then - echo $NEEDUPDATES - echo "" - read -p "Press U to update OS packages (recommended), C to continue without updates, or E to exit: " confirm - - if [[ "$confirm" == [cC] ]]; then + if [[ -z $UNATTENDED ]]; then + echo "$NEEDUPDATES" + echo "" + read -rp "Press U to update OS packages (recommended), C to continue without updates, or E to exit: " confirm + if [[ "$confirm" == [cC] ]]; then echo "Continuing without updating packages" - elif [[ "$confirm" == [uU] ]]; then + elif [[ "$confirm" == [uU] ]]; then echo "Applying Grid Updates" - set +e - run_check_net_err "salt '*' -b 5 state.apply patch.os queue=True" 'Could not apply OS updates, please check your network connection.' - set -e - else + update_flag=true + else echo "Exiting soup" exit 0 + fi + else + update_flag=true fi else - echo "Looks like you have an updated OS" + echo "Looks like you have an updated OS" + fi + + if [[ $update_flag == true ]]; then + set +e + run_check_net_err "salt '*' -b 5 state.apply patch.os queue=True" 'Could not apply OS updates, please check your network connection.' + set -e fi - } clean_dockers() { @@ -787,7 +793,7 @@ main() { echo "" check_airgap if [[ $is_airgap -eq 0 && $UNATTENDED == true && -z $ISOLOC ]]; then - echo "Missing file argument for unattended airgap upgrade." + echo "Missing file argument (-f ) for unattended airgap upgrade." exit 0 fi echo "Checking to see if this is a manager." From 9fb28709d5b9b8f57ae36c776727ae32d8784870 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 3 Sep 2021 10:47:00 -0400 Subject: [PATCH 51/51] Add maxfiles to the steno config --- salt/pcap/files/config | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/pcap/files/config b/salt/pcap/files/config index 048775ef7..900234bc1 100644 --- a/salt/pcap/files/config +++ b/salt/pcap/files/config @@ -1,11 +1,13 @@ {%- set interface = salt['pillar.get']('sensor:interface', 'bond0') %} {%- set diskfreepercentage = salt['pillar.get']('steno:diskfreepercentage', 10) %} +{%- set maxfiles = salt['pillar.get']('steno:maxfiles', 30000) %} + { "Threads": [ { "PacketsDirectory": "/nsm/pcap" , "IndexDirectory": "/nsm/pcapindex" - , "MaxDirectoryFiles": 30000 + , "MaxDirectoryFiles": {{ maxfiles }} , "DiskFreePercentage": {{ diskfreepercentage }} } ] @@ -15,4 +17,4 @@ , "Host": "127.0.0.1" , "Flags": ["-v", "--uid=stenographer", "--gid=stenographer"{{ BPF_COMPILED }}] , "CertPath": "/etc/stenographer/certs" -} \ No newline at end of file +}