From 9c2f7d574d36f41d5e1b948627747d7f25a15275 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 9 Jul 2020 11:19:02 -0400 Subject: [PATCH 01/12] Add ES settings to pillar --- .../templates/so/so-beats-template.json | 10 ++++++++++ .../templates/so/so-firewall-template.json | 10 ++++++++++ .../templates/so/so-ids-template.json | 10 ++++++++++ .../templates/so/so-import-template.json | 10 ++++++++++ .../templates/so/so-osquery-template.json | 10 ++++++++++ .../templates/so/so-ossec-template.json | 10 ++++++++++ .../templates/so/so-strelka-template.json | 10 ++++++++++ .../templates/so/so-syslog-template.json | 10 ++++++++++ setup/so-functions | 20 ++++++++++++++++++- 9 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 salt/logstash/pipelines/templates/so/so-beats-template.json create mode 100644 salt/logstash/pipelines/templates/so/so-firewall-template.json create mode 100644 salt/logstash/pipelines/templates/so/so-ids-template.json create mode 100644 salt/logstash/pipelines/templates/so/so-import-template.json create mode 100644 salt/logstash/pipelines/templates/so/so-osquery-template.json create mode 100644 salt/logstash/pipelines/templates/so/so-ossec-template.json create mode 100644 salt/logstash/pipelines/templates/so/so-strelka-template.json create mode 100644 salt/logstash/pipelines/templates/so/so-syslog-template.json diff --git a/salt/logstash/pipelines/templates/so/so-beats-template.json b/salt/logstash/pipelines/templates/so/so-beats-template.json new file mode 100644 index 000000000..48459bc7a --- /dev/null +++ b/salt/logstash/pipelines/templates/so/so-beats-template.json @@ -0,0 +1,10 @@ +{ + "index_patterns": ["so-beats-*"], + "version":50001, + "order" : 11, + "settings":{ + "number_of_replicas":0, + "number_of_shards":1, + "index.refresh_interval":"30s" + } +} diff --git a/salt/logstash/pipelines/templates/so/so-firewall-template.json b/salt/logstash/pipelines/templates/so/so-firewall-template.json new file mode 100644 index 000000000..61a95c0e7 --- /dev/null +++ b/salt/logstash/pipelines/templates/so/so-firewall-template.json @@ -0,0 +1,10 @@ +{ + "index_patterns": ["so-zeek-*"], + "version":50001, + "order" : 11, + "settings":{ + "number_of_replicas":0, + "number_of_shards":1, + "index.refresh_interval":"30s" + } +} diff --git a/salt/logstash/pipelines/templates/so/so-ids-template.json b/salt/logstash/pipelines/templates/so/so-ids-template.json new file mode 100644 index 000000000..61a95c0e7 --- /dev/null +++ b/salt/logstash/pipelines/templates/so/so-ids-template.json @@ -0,0 +1,10 @@ +{ + "index_patterns": ["so-zeek-*"], + "version":50001, + "order" : 11, + "settings":{ + "number_of_replicas":0, + "number_of_shards":1, + "index.refresh_interval":"30s" + } +} diff --git a/salt/logstash/pipelines/templates/so/so-import-template.json b/salt/logstash/pipelines/templates/so/so-import-template.json new file mode 100644 index 000000000..61a95c0e7 --- /dev/null +++ b/salt/logstash/pipelines/templates/so/so-import-template.json @@ -0,0 +1,10 @@ +{ + "index_patterns": ["so-zeek-*"], + "version":50001, + "order" : 11, + "settings":{ + "number_of_replicas":0, + "number_of_shards":1, + "index.refresh_interval":"30s" + } +} diff --git a/salt/logstash/pipelines/templates/so/so-osquery-template.json b/salt/logstash/pipelines/templates/so/so-osquery-template.json new file mode 100644 index 000000000..61a95c0e7 --- /dev/null +++ b/salt/logstash/pipelines/templates/so/so-osquery-template.json @@ -0,0 +1,10 @@ +{ + "index_patterns": ["so-zeek-*"], + "version":50001, + "order" : 11, + "settings":{ + "number_of_replicas":0, + "number_of_shards":1, + "index.refresh_interval":"30s" + } +} diff --git a/salt/logstash/pipelines/templates/so/so-ossec-template.json b/salt/logstash/pipelines/templates/so/so-ossec-template.json new file mode 100644 index 000000000..61a95c0e7 --- /dev/null +++ b/salt/logstash/pipelines/templates/so/so-ossec-template.json @@ -0,0 +1,10 @@ +{ + "index_patterns": ["so-zeek-*"], + "version":50001, + "order" : 11, + "settings":{ + "number_of_replicas":0, + "number_of_shards":1, + "index.refresh_interval":"30s" + } +} diff --git a/salt/logstash/pipelines/templates/so/so-strelka-template.json b/salt/logstash/pipelines/templates/so/so-strelka-template.json new file mode 100644 index 000000000..61a95c0e7 --- /dev/null +++ b/salt/logstash/pipelines/templates/so/so-strelka-template.json @@ -0,0 +1,10 @@ +{ + "index_patterns": ["so-zeek-*"], + "version":50001, + "order" : 11, + "settings":{ + "number_of_replicas":0, + "number_of_shards":1, + "index.refresh_interval":"30s" + } +} diff --git a/salt/logstash/pipelines/templates/so/so-syslog-template.json b/salt/logstash/pipelines/templates/so/so-syslog-template.json new file mode 100644 index 000000000..61a95c0e7 --- /dev/null +++ b/salt/logstash/pipelines/templates/so/so-syslog-template.json @@ -0,0 +1,10 @@ +{ + "index_patterns": ["so-zeek-*"], + "version":50001, + "order" : 11, + "settings":{ + "number_of_replicas":0, + "number_of_shards":1, + "index.refresh_interval":"30s" + } +} diff --git a/setup/so-functions b/setup/so-functions index 5bbb319eb..68bd349bd 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1068,10 +1068,28 @@ elasticsearch_pillar() { " log_size_limit: $log_size_limit"\ " cur_close_days: $CURCLOSEDAYS"\ " route_type: hot"\ + " replicas: 0"\ + " true_cluster: False" + " true_cluster_name: so" " index_settings:"\ + " so-beats:"\ + " shards: 1"\ + " so-firewall:"\ + " shards: 1"\ + " so-ids:"\ + " shards: 1"\ + " so-import:"\ + " shards: 1"\ + " so-osquery:"\ + " shards: 1"\ + " so-ossec:"\ + " shards: 1"\ + " so-strelka:"\ + " shards: 1"\ + " so-syslog:"\ + " shards: 1"\ " so-zeek:"\ " shards: 5"\ - " replicas: 0"\ "" >> "$pillar_file" if [ "$install_type" != 'EVAL' ] && [ "$install_type" != 'HELIXSENSOR' ] && [ "$install_type" != 'MASTERSEARCH' ] && [ "$install_type" != 'STANDALONE' ]; then From 96bcf9d9f3b5f878095d43fc418cf5b1a4336ed7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 9 Jul 2020 11:51:55 -0400 Subject: [PATCH 02/12] Add temaplte files per index --- .../pipelines/templates/so/so-beats-template.json | 13 ++++++++----- .../pipelines/templates/so/so-common-template.json | 8 ++++---- .../templates/so/so-firewall-template.json | 13 ++++++++----- .../pipelines/templates/so/so-ids-template.json | 13 ++++++++----- .../pipelines/templates/so/so-import-template.json | 13 ++++++++----- .../templates/so/so-osquery-template.json | 13 ++++++++----- .../pipelines/templates/so/so-ossec-template.json | 13 ++++++++----- .../templates/so/so-strelka-template.json | 13 ++++++++----- .../pipelines/templates/so/so-syslog-template.json | 14 +++++++++----- .../pipelines/templates/so/so-zeek-template.json | 11 +++++++---- 10 files changed, 76 insertions(+), 48 deletions(-) diff --git a/salt/logstash/pipelines/templates/so/so-beats-template.json b/salt/logstash/pipelines/templates/so/so-beats-template.json index 48459bc7a..6d2cf7851 100644 --- a/salt/logstash/pipelines/templates/so/so-beats-template.json +++ b/salt/logstash/pipelines/templates/so/so-beats-template.json @@ -1,10 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-beats:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-beats:refresh', '30s') %} { "index_patterns": ["so-beats-*"], - "version":50001, - "order" : 11, + "version": 50001, + "order": 11, "settings":{ - "number_of_replicas":0, - "number_of_shards":1, - "index.refresh_interval":"30s" + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" } } diff --git a/salt/logstash/pipelines/templates/so/so-common-template.json b/salt/logstash/pipelines/templates/so/so-common-template.json index 396e26c3c..1b4bb1206 100644 --- a/salt/logstash/pipelines/templates/so/so-common-template.json +++ b/salt/logstash/pipelines/templates/so/so-common-template.json @@ -1,15 +1,15 @@ { "index_patterns": ["so-ids-*", "so-firewall-*", "so-syslog-*", "so-zeek-*", "so-import-*", "so-ossec-*", "so-strelka-*", "so-beats-*", "so-osquery-*"], "version":50001, - "order" : 10, + "order":10, "settings":{ "number_of_replicas":0, "number_of_shards":1, "index.refresh_interval":"30s" }, "mappings":{ - "dynamic": false, - "date_detection": false, + "dynamic":false, + "date_detection":false, "properties":{ "@timestamp":{ "type":"date" @@ -19,7 +19,7 @@ }, "osquery":{ "type":"object", - "dynamic": true + "dynamic":true }, "geoip":{ "dynamic":true, diff --git a/salt/logstash/pipelines/templates/so/so-firewall-template.json b/salt/logstash/pipelines/templates/so/so-firewall-template.json index 61a95c0e7..7bc81fd12 100644 --- a/salt/logstash/pipelines/templates/so/so-firewall-template.json +++ b/salt/logstash/pipelines/templates/so/so-firewall-template.json @@ -1,10 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-firewall:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-firewall:refresh', '30s') %} { - "index_patterns": ["so-zeek-*"], + "index_patterns": ["so-firewall-*"], "version":50001, - "order" : 11, + "order":11, "settings":{ - "number_of_replicas":0, - "number_of_shards":1, - "index.refresh_interval":"30s" + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" } } diff --git a/salt/logstash/pipelines/templates/so/so-ids-template.json b/salt/logstash/pipelines/templates/so/so-ids-template.json index 61a95c0e7..abf37319a 100644 --- a/salt/logstash/pipelines/templates/so/so-ids-template.json +++ b/salt/logstash/pipelines/templates/so/so-ids-template.json @@ -1,10 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-ids:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-ids:refresh', '30s') %} { - "index_patterns": ["so-zeek-*"], + "index_patterns": ["so-ids-*"], "version":50001, - "order" : 11, + "order":11, "settings":{ - "number_of_replicas":0, - "number_of_shards":1, - "index.refresh_interval":"30s" + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" } } diff --git a/salt/logstash/pipelines/templates/so/so-import-template.json b/salt/logstash/pipelines/templates/so/so-import-template.json index 61a95c0e7..e4d68235d 100644 --- a/salt/logstash/pipelines/templates/so/so-import-template.json +++ b/salt/logstash/pipelines/templates/so/so-import-template.json @@ -1,10 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-import:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-import:refresh', '30s') %} { - "index_patterns": ["so-zeek-*"], + "index_patterns": ["so-import-*"], "version":50001, - "order" : 11, + "order": 11, "settings":{ - "number_of_replicas":0, - "number_of_shards":1, - "index.refresh_interval":"30s" + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" } } diff --git a/salt/logstash/pipelines/templates/so/so-osquery-template.json b/salt/logstash/pipelines/templates/so/so-osquery-template.json index 61a95c0e7..47cb3ebab 100644 --- a/salt/logstash/pipelines/templates/so/so-osquery-template.json +++ b/salt/logstash/pipelines/templates/so/so-osquery-template.json @@ -1,10 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-osquery:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-osquery:refresh', '30s') %} { - "index_patterns": ["so-zeek-*"], + "index_patterns": ["so-osquery-*"], "version":50001, - "order" : 11, + "order":11, "settings":{ - "number_of_replicas":0, - "number_of_shards":1, - "index.refresh_interval":"30s" + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" } } diff --git a/salt/logstash/pipelines/templates/so/so-ossec-template.json b/salt/logstash/pipelines/templates/so/so-ossec-template.json index 61a95c0e7..ce903e228 100644 --- a/salt/logstash/pipelines/templates/so/so-ossec-template.json +++ b/salt/logstash/pipelines/templates/so/so-ossec-template.json @@ -1,10 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-ossec:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-ossec:refresh', '30s') %} { - "index_patterns": ["so-zeek-*"], + "index_patterns": ["so-ossec-*"], "version":50001, - "order" : 11, + "order":11, "settings":{ - "number_of_replicas":0, - "number_of_shards":1, - "index.refresh_interval":"30s" + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" } } diff --git a/salt/logstash/pipelines/templates/so/so-strelka-template.json b/salt/logstash/pipelines/templates/so/so-strelka-template.json index 61a95c0e7..2f7db541a 100644 --- a/salt/logstash/pipelines/templates/so/so-strelka-template.json +++ b/salt/logstash/pipelines/templates/so/so-strelka-template.json @@ -1,10 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-strelka:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-strelka:refresh', '30s') %} { - "index_patterns": ["so-zeek-*"], + "index_patterns": ["so-strelka-*"], "version":50001, - "order" : 11, + "order":11, "settings":{ - "number_of_replicas":0, - "number_of_shards":1, - "index.refresh_interval":"30s" + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" } } diff --git a/salt/logstash/pipelines/templates/so/so-syslog-template.json b/salt/logstash/pipelines/templates/so/so-syslog-template.json index 61a95c0e7..47f8d78e6 100644 --- a/salt/logstash/pipelines/templates/so/so-syslog-template.json +++ b/salt/logstash/pipelines/templates/so/so-syslog-template.json @@ -1,10 +1,14 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-syslog:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-syslog:refresh', '30s') %} { - "index_patterns": ["so-zeek-*"], + "index_patterns": ["so-syslog-*"], "version":50001, - "order" : 11, + "order":11, "settings":{ - "number_of_replicas":0, - "number_of_shards":1, - "index.refresh_interval":"30s" + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" } } + diff --git a/salt/logstash/pipelines/templates/so/so-zeek-template.json b/salt/logstash/pipelines/templates/so/so-zeek-template.json index 61a95c0e7..616607f52 100644 --- a/salt/logstash/pipelines/templates/so/so-zeek-template.json +++ b/salt/logstash/pipelines/templates/so/so-zeek-template.json @@ -1,10 +1,13 @@ +{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:shards', 1) %} +{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %} +{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zeek:refresh', '30s') %} { "index_patterns": ["so-zeek-*"], "version":50001, - "order" : 11, + "order":11, "settings":{ - "number_of_replicas":0, - "number_of_shards":1, - "index.refresh_interval":"30s" + "number_of_replicas":{{ REPLICAS }}, + "number_of_shards":{{ SHARDS }}, + "index.refresh_interval":"{{ REFRESH }}" } } From ca20279a09df2422cf636018fd0f2487e3bb62f5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 9 Jul 2020 12:00:07 -0400 Subject: [PATCH 03/12] Add curator to static pillar --- setup/so-functions | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/setup/so-functions b/setup/so-functions index 68bd349bd..42f4f809b 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1013,6 +1013,11 @@ master_static() { "strelka:"\ " enabled: $STRELKA"\ " rules: $STRELKARULES"\ + "curator:"\ + " hot_warm: False"\ + " warm: 7"\ + " close: 30" + " delete: 45" "elastic:"\ " features: False" > "$static_pillar" From 357efac8739a06f6fa177c1338e24c9c3956739f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 9 Jul 2020 12:10:53 -0400 Subject: [PATCH 04/12] Add index specific curator settings --- setup/so-functions | 80 ++++++++++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 28 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 42f4f809b..df3459872 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1015,11 +1015,59 @@ master_static() { " rules: $STRELKARULES"\ "curator:"\ " hot_warm: False"\ - " warm: 7"\ - " close: 30" - " delete: 45" "elastic:"\ - " features: False" > "$static_pillar" + " features: False"\ + "elasticsearch:"\ + " route_type: hot"\ + " replicas: 0"\ + " true_cluster: False" + " true_cluster_name: so" + " index_settings:"\ + " so-beats:"\ + " shards: 1"\ + " warm: 7"\ + " close: 30"\ + " delete: 45" + " so-firewall:"\ + " warm: 7"\ + " close: 30"\ + " delete: 45" + " shards: 1"\ + " so-ids:"\ + " shards: 1"\ + " warm: 7"\ + " close: 30"\ + " delete: 45" + " so-import:"\ + " warm: 7"\ + " close: 30"\ + " delete: 45" + " shards: 1"\ + " so-osquery:"\ + " shards: 1"\ + " warm: 7"\ + " close: 30"\ + " delete: 45" + " so-ossec:"\ + " shards: 1"\ + " warm: 7"\ + " close: 30"\ + " delete: 45" + " so-strelka:"\ + " shards: 1"\ + " warm: 7"\ + " close: 30"\ + " delete: 45" + " so-syslog:"\ + " shards: 1"\ + " warm: 7"\ + " close: 30"\ + " delete: 45" + " so-zeek:"\ + " shards: 5"\ + " warm: 7"\ + " close: 30"\ + " delete: 45" > "$static_pillar" printf '%s\n' '----' >> "$setup_log" 2>&1 cat "$static_pillar" >> "$setup_log" 2>&1 @@ -1071,30 +1119,6 @@ elasticsearch_pillar() { " node_type: $NODETYPE"\ " es_port: $node_es_port"\ " log_size_limit: $log_size_limit"\ - " cur_close_days: $CURCLOSEDAYS"\ - " route_type: hot"\ - " replicas: 0"\ - " true_cluster: False" - " true_cluster_name: so" - " index_settings:"\ - " so-beats:"\ - " shards: 1"\ - " so-firewall:"\ - " shards: 1"\ - " so-ids:"\ - " shards: 1"\ - " so-import:"\ - " shards: 1"\ - " so-osquery:"\ - " shards: 1"\ - " so-ossec:"\ - " shards: 1"\ - " so-strelka:"\ - " shards: 1"\ - " so-syslog:"\ - " shards: 1"\ - " so-zeek:"\ - " shards: 5"\ "" >> "$pillar_file" if [ "$install_type" != 'EVAL' ] && [ "$install_type" != 'HELIXSENSOR' ] && [ "$install_type" != 'MASTERSEARCH' ] && [ "$install_type" != 'STANDALONE' ]; then From 7c6677916a435dff2e44b74a7c222641b7a13024 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 9 Jul 2020 12:56:29 -0400 Subject: [PATCH 05/12] Curator actions --- salt/curator/files/action/delete.yml | 6 +--- salt/curator/files/action/so-beats-close.yml | 29 +++++++++++++++++++ .../{close.yml => so-firewall-close.yml} | 12 ++------ salt/curator/files/action/so-ids-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-import-close.yml | 29 +++++++++++++++++++ .../curator/files/action/so-osquery-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-ossec-close.yml | 29 +++++++++++++++++++ .../curator/files/action/so-strelka-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-syslog-close.yml | 29 +++++++++++++++++++ salt/curator/files/action/so-zeek-close.yml | 29 +++++++++++++++++++ setup/so-functions | 4 +-- 11 files changed, 238 insertions(+), 16 deletions(-) create mode 100644 salt/curator/files/action/so-beats-close.yml rename salt/curator/files/action/{close.yml => so-firewall-close.yml} (58%) create mode 100644 salt/curator/files/action/so-ids-close.yml create mode 100644 salt/curator/files/action/so-import-close.yml create mode 100644 salt/curator/files/action/so-osquery-close.yml create mode 100644 salt/curator/files/action/so-ossec-close.yml create mode 100644 salt/curator/files/action/so-strelka-close.yml create mode 100644 salt/curator/files/action/so-syslog-close.yml create mode 100644 salt/curator/files/action/so-zeek-close.yml diff --git a/salt/curator/files/action/delete.yml b/salt/curator/files/action/delete.yml index f24f0b781..fb3945c1d 100644 --- a/salt/curator/files/action/delete.yml +++ b/salt/curator/files/action/delete.yml @@ -1,8 +1,4 @@ -{%- if grains['role'] in ['so-node', 'so-searchnode', 'so-heavynode'] %} - {%- set log_size_limit = salt['pillar.get']('elasticsearch:log_size_limit', '') -%} -{%- elif grains['role'] in ['so-eval', 'so-mastersearch', 'so-standalone'] %} - {%- set log_size_limit = salt['pillar.get']('master:log_size_limit', '') -%} -{%- endif %} +{%- set log_size_limit = salt['pillar.get']('elasticsearch:log_size_limit', '') -%} --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" diff --git a/salt/curator/files/action/so-beats-close.yml b/salt/curator/files/action/so-beats-close.yml new file mode 100644 index 000000000..dbbcca1c8 --- /dev/null +++ b/salt/curator/files/action/so-beats-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settins:so-beats:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close Beats indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-beats.*|so-beats.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/close.yml b/salt/curator/files/action/so-firewall-close.yml similarity index 58% rename from salt/curator/files/action/close.yml rename to salt/curator/files/action/so-firewall-close.yml index d0bd1d5d1..46f0b39a9 100644 --- a/salt/curator/files/action/close.yml +++ b/salt/curator/files/action/so-firewall-close.yml @@ -1,9 +1,4 @@ -{%- if grains['role'] in ['so-node', 'so-searchnode', 'so-heavynode'] %} - {%- set cur_close_days = salt['pillar.get']('elasticsearch:cur_close_days', '') -%} -{%- elif grains['role'] in ['so-eval', 'so-mastersearch', 'so-standalone'] %} - {%- set cur_close_days = salt['pillar.get']('master:cur_close_days', '') -%} -{%- endif -%} - +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settins:so-firewall:close', 30) -%} --- # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" @@ -15,8 +10,7 @@ actions: 1: action: close description: >- - Close indices older than {{cur_close_days}} days (based on index name), for logstash- - prefixed indices. + Close Firewall indices older than {{cur_close_days}} days. options: delete_aliases: False timeout_override: @@ -25,7 +19,7 @@ actions: filters: - filtertype: pattern kind: regex - value: '^(logstash-.*|so-.*)$' + value: '^(logstash-firewall.*|so-firewall.*)$' - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/so-ids-close.yml b/salt/curator/files/action/so-ids-close.yml new file mode 100644 index 000000000..89f08d8d1 --- /dev/null +++ b/salt/curator/files/action/so-ids-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settins:so-ids:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close IDS indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-ids.*|so-ids.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-import-close.yml b/salt/curator/files/action/so-import-close.yml new file mode 100644 index 000000000..b9ee6e5da --- /dev/null +++ b/salt/curator/files/action/so-import-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settins:so-import:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close Import indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-import.*|so-import.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-osquery-close.yml b/salt/curator/files/action/so-osquery-close.yml new file mode 100644 index 000000000..152a41afa --- /dev/null +++ b/salt/curator/files/action/so-osquery-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settins:so-osquery:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close osquery indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-osquery.*|so-osquery.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-ossec-close.yml b/salt/curator/files/action/so-ossec-close.yml new file mode 100644 index 000000000..5ee8c91de --- /dev/null +++ b/salt/curator/files/action/so-ossec-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settins:so-ossec:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close ossec indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-ossec.*|so-ossec.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-strelka-close.yml b/salt/curator/files/action/so-strelka-close.yml new file mode 100644 index 000000000..a07ab94e8 --- /dev/null +++ b/salt/curator/files/action/so-strelka-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settins:so-strelka:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close Strelka indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-strelka.*|so-strelka.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-syslog-close.yml b/salt/curator/files/action/so-syslog-close.yml new file mode 100644 index 000000000..3aae50566 --- /dev/null +++ b/salt/curator/files/action/so-syslog-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settins:so-syslog:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close syslog indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-syslog.*|so-syslog.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/so-zeek-close.yml b/salt/curator/files/action/so-zeek-close.yml new file mode 100644 index 000000000..ec1ab9eff --- /dev/null +++ b/salt/curator/files/action/so-zeek-close.yml @@ -0,0 +1,29 @@ +{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settins:so-zeek:close', 30) -%} +--- +# Remember, leave a key empty if there is no value. None will be a string, +# not a Python "NoneType" +# +# Also remember that all examples have 'disable_action' set to True. If you +# want to use this action as a template, be sure to set this to False after +# copying it. +actions: + 1: + action: close + description: >- + Close Zeek indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(logstash-zeek.*|so-zeek.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/setup/so-functions b/setup/so-functions index df3459872..6d71fbe44 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1040,8 +1040,8 @@ master_static() { " delete: 45" " so-import:"\ " warm: 7"\ - " close: 30"\ - " delete: 45" + " close: 7300"\ + " delete: 7301" " shards: 1"\ " so-osquery:"\ " shards: 1"\ From 3c6465bb7f5347ce72f6ebe79fdd55954b8b9ea1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 9 Jul 2020 16:42:39 -0400 Subject: [PATCH 06/12] ES Jinja the config --- salt/elasticsearch/files/elasticsearch.yml | 24 +++++++-------- setup/so-functions | 36 +++++++++++++--------- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index 02dd42aa5..8833f801e 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -1,6 +1,11 @@ -{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' %} -{%- set esclustername = salt['pillar.get']('master:esclustername', '') %} -cluster.name: "{{ esclustername }}" +{%- set NODE_ROUTE_TYPE = salt['pillar.get']('elasticsearch:node_route_type', 'hot') %} +{%- if salt['pillar.get']('elasticsearch:hot_warm_enabled') or if salt['pillar.get']('elasticsearch:true_cluster')} +{%- set ESCLUSTERNAME = salt['pillar.get']('elasticsearch:true_cluster_name', '') %} +{%- else %} +{%- set ESCLUSTERNAME = salt['pillar.get']('elasticsearch:esclustername', '') %} +{%- endif %} +{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} +cluster.name: "{{ ESCLUSTERNAME }}" network.host: 0.0.0.0 # minimum_master_nodes need to be explicitly set when bound on a public IP @@ -10,19 +15,12 @@ discovery.zen.minimum_master_nodes: 1 # This is a test -- if this is here, then the volume is mounted correctly. path.logs: /var/log/elasticsearch action.destructive_requires_name: true -{%- else %} -{%- set esclustername = salt['grains.get']('host', '') %} -{%- set nodeip = salt['pillar.get']('elasticsearch:mainip', '') -%} -cluster.name: "{{ esclustername }}" -network.host: 0.0.0.0 -discovery.zen.minimum_master_nodes: 1 -path.logs: /var/log/elasticsearch -action.destructive_requires_name: true transport.bind_host: 0.0.0.0 -transport.publish_host: {{ nodeip }} +transport.publish_host: {{ NODEIP }} transport.publish_port: 9300 -{%- endif %} cluster.routing.allocation.disk.threshold_enabled: true cluster.routing.allocation.disk.watermark.low: 95% cluster.routing.allocation.disk.watermark.high: 98% cluster.routing.allocation.disk.watermark.flood_stage: 98% +node.attr.box_type: {{ NODE_ROUTE_TYPE }} +node.name: {{ esclustername }} diff --git a/setup/so-functions b/setup/so-functions index 6d71fbe44..8ee44ab1d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1018,55 +1018,60 @@ master_static() { "elastic:"\ " features: False"\ "elasticsearch:"\ - " route_type: hot"\ " replicas: 0"\ - " true_cluster: False" - " true_cluster_name: so" + " true_cluster: False"\ + " true_cluster_name: so"\ + " discovery_nodes: 1"\ + " hot_warm_enabled: False"\ + " cluster_routing_allocation_disk.threshold_enabled: true"\ + " cluster_routing_allocation_disk_watermark_low: 95%"\ + " cluster_routing_allocation_disk_watermark_high: 98%"\ + " cluster_routing_allocation_disk_watermark_flood_stage: 98%"\ " index_settings:"\ " so-beats:"\ " shards: 1"\ " warm: 7"\ " close: 30"\ - " delete: 45" + " delete: 365" " so-firewall:"\ + " shards: 1"\ " warm: 7"\ " close: 30"\ - " delete: 45" - " shards: 1"\ + " delete: 365"\ " so-ids:"\ " shards: 1"\ " warm: 7"\ " close: 30"\ - " delete: 45" + " delete: 365"\ " so-import:"\ + " shards: 1"\ " warm: 7"\ - " close: 7300"\ - " delete: 7301" - " shards: 1"\ + " close: 73000"\ + " delete: 73001" " so-osquery:"\ " shards: 1"\ " warm: 7"\ " close: 30"\ - " delete: 45" + " delete: 365" " so-ossec:"\ " shards: 1"\ " warm: 7"\ " close: 30"\ - " delete: 45" + " delete: 365"\ " so-strelka:"\ " shards: 1"\ " warm: 7"\ " close: 30"\ - " delete: 45" + " delete: 365"\ " so-syslog:"\ " shards: 1"\ " warm: 7"\ " close: 30"\ - " delete: 45" + " delete: 365"\ " so-zeek:"\ " shards: 5"\ " warm: 7"\ - " close: 30"\ + " close: 365"\ " delete: 45" > "$static_pillar" printf '%s\n' '----' >> "$setup_log" 2>&1 @@ -1119,6 +1124,7 @@ elasticsearch_pillar() { " node_type: $NODETYPE"\ " es_port: $node_es_port"\ " log_size_limit: $log_size_limit"\ + " node_route_type: hot"\ "" >> "$pillar_file" if [ "$install_type" != 'EVAL' ] && [ "$install_type" != 'HELIXSENSOR' ] && [ "$install_type" != 'MASTERSEARCH' ] && [ "$install_type" != 'STANDALONE' ]; then From 6094d19b0bfec70c7b511c3ed5a26b4c8b625b2e Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 9 Jul 2020 16:54:31 -0400 Subject: [PATCH 07/12] Make hot default --- salt/elasticsearch/files/elasticsearch.yml | 2 +- salt/logstash/pipelines/templates/so/so-common-template.json | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index 8833f801e..ac27fb9b5 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -1,5 +1,5 @@ {%- set NODE_ROUTE_TYPE = salt['pillar.get']('elasticsearch:node_route_type', 'hot') %} -{%- if salt['pillar.get']('elasticsearch:hot_warm_enabled') or if salt['pillar.get']('elasticsearch:true_cluster')} +{%- if salt['pillar.get']('elasticsearch:hot_warm_enabled') or salt['pillar.get']('elasticsearch:true_cluster')} {%- set ESCLUSTERNAME = salt['pillar.get']('elasticsearch:true_cluster_name', '') %} {%- else %} {%- set ESCLUSTERNAME = salt['pillar.get']('elasticsearch:esclustername', '') %} diff --git a/salt/logstash/pipelines/templates/so/so-common-template.json b/salt/logstash/pipelines/templates/so/so-common-template.json index 1b4bb1206..a4da40765 100644 --- a/salt/logstash/pipelines/templates/so/so-common-template.json +++ b/salt/logstash/pipelines/templates/so/so-common-template.json @@ -5,7 +5,8 @@ "settings":{ "number_of_replicas":0, "number_of_shards":1, - "index.refresh_interval":"30s" + "index.refresh_interval":"30s", + "index.routing.allocation.require.box_type":"hot" }, "mappings":{ "dynamic":false, From ad6c9e7fe9349962ab13772120ff1c0167531632 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 9 Jul 2020 16:58:35 -0400 Subject: [PATCH 08/12] recurse actions for curator --- salt/curator/init.sls | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 8d3147242..049a1cae8 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -30,18 +30,10 @@ curlogdir: - user: 934 - group: 939 -curcloseconf: - file.managed: - - name: /opt/so/conf/curator/action/close.yml - - source: salt://curator/files/action/close.yml - - user: 934 - - group: 939 - - template: jinja - -curdelconf: - file.managed: - - name: /opt/so/conf/curator/action/delete.yml - - source: salt://curator/files/action/delete.yml +actionconfs: + file.recurse: + - name: /opt/so/conf/curator/action + - source: salt://curator/files/action - user: 934 - group: 939 - template: jinja From 9a7035326dc49e3e6341a7750c5cf4720a2bcbf5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 9 Jul 2020 17:09:20 -0400 Subject: [PATCH 09/12] Update Logstash pillar --- pillar/logstash/eval.sls | 7 +++++++ pillar/logstash/search.sls | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/pillar/logstash/eval.sls b/pillar/logstash/eval.sls index 39a87dc77..e76f16d72 100644 --- a/pillar/logstash/eval.sls +++ b/pillar/logstash/eval.sls @@ -18,4 +18,11 @@ logstash: templates: - so/so-beats-template.json - so/so-common-template.json + - so/so-firewall-template.json + - so/so-ids-template.json + - so/so-import-template.json + - so/so-osquery-template.json + - so/so-ossec-template.json + - so/so-strelka-template.json + - so/so-syslog-template.json - so/so-zeek-template.json diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 6b3d0422e..033243956 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -12,5 +12,13 @@ logstash: - so/9600_output_ossec.conf.jinja - so/9700_output_strelka.conf.jinja templates: + - so/so-beats-template.json - so/so-common-template.json + - so/so-firewall-template.json + - so/so-ids-template.json + - so/so-import-template.json + - so/so-osquery-template.json + - so/so-ossec-template.json + - so/so-strelka-template.json + - so/so-syslog-template.json - so/so-zeek-template.json From 8ef18f90443df2401ad5cde7f714a04deb509472 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 9 Jul 2020 18:51:59 -0400 Subject: [PATCH 10/12] Fiz pillar --- setup/so-functions | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 8ee44ab1d..51d6d43e6 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1032,7 +1032,7 @@ master_static() { " shards: 1"\ " warm: 7"\ " close: 30"\ - " delete: 365" + " delete: 365"\ " so-firewall:"\ " shards: 1"\ " warm: 7"\ @@ -1044,15 +1044,15 @@ master_static() { " close: 30"\ " delete: 365"\ " so-import:"\ - " shards: 1"\ + " shards: 1"\ " warm: 7"\ " close: 73000"\ - " delete: 73001" + " delete: 73001"\ " so-osquery:"\ " shards: 1"\ " warm: 7"\ " close: 30"\ - " delete: 365" + " delete: 365"\ " so-ossec:"\ " shards: 1"\ " warm: 7"\ From 1a6c4c12b4a568ea6815a95542df1101e77296b6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 9 Jul 2020 21:56:32 -0400 Subject: [PATCH 11/12] Fix elasticsearch yaml --- salt/elasticsearch/files/elasticsearch.yml | 4 ++-- .../{so-beats-template.json => so-beats-template.json.jinja} | 0 ...firewall-template.json => so-firewall-template.json.jinja} | 0 .../so/{so-ids-template.json => so-ids-template.json.jinja} | 0 ...{so-import-template.json => so-import-template.json.jinja} | 0 5 files changed, 2 insertions(+), 2 deletions(-) rename salt/logstash/pipelines/templates/so/{so-beats-template.json => so-beats-template.json.jinja} (100%) rename salt/logstash/pipelines/templates/so/{so-firewall-template.json => so-firewall-template.json.jinja} (100%) rename salt/logstash/pipelines/templates/so/{so-ids-template.json => so-ids-template.json.jinja} (100%) rename salt/logstash/pipelines/templates/so/{so-import-template.json => so-import-template.json.jinja} (100%) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index ac27fb9b5..a38c3db87 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -1,5 +1,5 @@ {%- set NODE_ROUTE_TYPE = salt['pillar.get']('elasticsearch:node_route_type', 'hot') %} -{%- if salt['pillar.get']('elasticsearch:hot_warm_enabled') or salt['pillar.get']('elasticsearch:true_cluster')} +{%- if salt['pillar.get']('elasticsearch:hot_warm_enabled') or salt['pillar.get']('elasticsearch:true_cluster') %} {%- set ESCLUSTERNAME = salt['pillar.get']('elasticsearch:true_cluster_name', '') %} {%- else %} {%- set ESCLUSTERNAME = salt['pillar.get']('elasticsearch:esclustername', '') %} @@ -23,4 +23,4 @@ cluster.routing.allocation.disk.watermark.low: 95% cluster.routing.allocation.disk.watermark.high: 98% cluster.routing.allocation.disk.watermark.flood_stage: 98% node.attr.box_type: {{ NODE_ROUTE_TYPE }} -node.name: {{ esclustername }} +node.name: {{ ESCLUSTERNAME }} diff --git a/salt/logstash/pipelines/templates/so/so-beats-template.json b/salt/logstash/pipelines/templates/so/so-beats-template.json.jinja similarity index 100% rename from salt/logstash/pipelines/templates/so/so-beats-template.json rename to salt/logstash/pipelines/templates/so/so-beats-template.json.jinja diff --git a/salt/logstash/pipelines/templates/so/so-firewall-template.json b/salt/logstash/pipelines/templates/so/so-firewall-template.json.jinja similarity index 100% rename from salt/logstash/pipelines/templates/so/so-firewall-template.json rename to salt/logstash/pipelines/templates/so/so-firewall-template.json.jinja diff --git a/salt/logstash/pipelines/templates/so/so-ids-template.json b/salt/logstash/pipelines/templates/so/so-ids-template.json.jinja similarity index 100% rename from salt/logstash/pipelines/templates/so/so-ids-template.json rename to salt/logstash/pipelines/templates/so/so-ids-template.json.jinja diff --git a/salt/logstash/pipelines/templates/so/so-import-template.json b/salt/logstash/pipelines/templates/so/so-import-template.json.jinja similarity index 100% rename from salt/logstash/pipelines/templates/so/so-import-template.json rename to salt/logstash/pipelines/templates/so/so-import-template.json.jinja From 3706aa76d8b6b26152d86d73a86d7c2ace5b93ba Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 10 Jul 2020 10:35:31 -0400 Subject: [PATCH 12/12] Add jinja extension --- pillar/logstash/search.sls | 18 +++++++++--------- ...ate.json => so-osquery-template.json.jinja} | 0 ...plate.json => so-ossec-template.json.jinja} | 0 ...ate.json => so-strelka-template.json.jinja} | 0 ...late.json => so-syslog-template.json.jinja} | 0 ...mplate.json => so-zeek-template.json.jinja} | 0 6 files changed, 9 insertions(+), 9 deletions(-) rename salt/logstash/pipelines/templates/so/{so-osquery-template.json => so-osquery-template.json.jinja} (100%) rename salt/logstash/pipelines/templates/so/{so-ossec-template.json => so-ossec-template.json.jinja} (100%) rename salt/logstash/pipelines/templates/so/{so-strelka-template.json => so-strelka-template.json.jinja} (100%) rename salt/logstash/pipelines/templates/so/{so-syslog-template.json => so-syslog-template.json.jinja} (100%) rename salt/logstash/pipelines/templates/so/{so-zeek-template.json => so-zeek-template.json.jinja} (100%) diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 033243956..6602e0591 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -12,13 +12,13 @@ logstash: - so/9600_output_ossec.conf.jinja - so/9700_output_strelka.conf.jinja templates: - - so/so-beats-template.json + - so/so-beats-template.json.jinja - so/so-common-template.json - - so/so-firewall-template.json - - so/so-ids-template.json - - so/so-import-template.json - - so/so-osquery-template.json - - so/so-ossec-template.json - - so/so-strelka-template.json - - so/so-syslog-template.json - - so/so-zeek-template.json + - so/so-firewall-template.json.jinja + - so/so-ids-template.json.jinja + - so/so-import-template.json.jinja + - so/so-osquery-template.json.jinja + - so/so-ossec-template.json.jinja + - so/so-strelka-template.json.jinja + - so/so-syslog-template.json.jinja + - so/so-zeek-template.json.jinja diff --git a/salt/logstash/pipelines/templates/so/so-osquery-template.json b/salt/logstash/pipelines/templates/so/so-osquery-template.json.jinja similarity index 100% rename from salt/logstash/pipelines/templates/so/so-osquery-template.json rename to salt/logstash/pipelines/templates/so/so-osquery-template.json.jinja diff --git a/salt/logstash/pipelines/templates/so/so-ossec-template.json b/salt/logstash/pipelines/templates/so/so-ossec-template.json.jinja similarity index 100% rename from salt/logstash/pipelines/templates/so/so-ossec-template.json rename to salt/logstash/pipelines/templates/so/so-ossec-template.json.jinja diff --git a/salt/logstash/pipelines/templates/so/so-strelka-template.json b/salt/logstash/pipelines/templates/so/so-strelka-template.json.jinja similarity index 100% rename from salt/logstash/pipelines/templates/so/so-strelka-template.json rename to salt/logstash/pipelines/templates/so/so-strelka-template.json.jinja diff --git a/salt/logstash/pipelines/templates/so/so-syslog-template.json b/salt/logstash/pipelines/templates/so/so-syslog-template.json.jinja similarity index 100% rename from salt/logstash/pipelines/templates/so/so-syslog-template.json rename to salt/logstash/pipelines/templates/so/so-syslog-template.json.jinja diff --git a/salt/logstash/pipelines/templates/so/so-zeek-template.json b/salt/logstash/pipelines/templates/so/so-zeek-template.json.jinja similarity index 100% rename from salt/logstash/pipelines/templates/so/so-zeek-template.json rename to salt/logstash/pipelines/templates/so/so-zeek-template.json.jinja