Merge pull request #7206 from Security-Onion-Solutions/feature/template-reorg

Re-organize Elasticsearch Index Templates

pillar/elasticsearch/eval.sls

@@ -1,14 +1,2 @@
 elasticsearch:
   templates:
-    - so/so-beats-template.json.jinja
-    - so/so-case-template.json.jinja
-    - so/so-common-template.json.jinja
-    - so/so-firewall-template.json.jinja
-    - so/so-flow-template.json.jinja
-    - so/so-ids-template.json.jinja
-    - so/so-import-template.json.jinja
-    - so/so-osquery-template.json.jinja
-    - so/so-ossec-template.json.jinja
-    - so/so-strelka-template.json.jinja
-    - so/so-syslog-template.json.jinja
-    - so/so-zeek-template.json.jinja

pillar/elasticsearch/manager.sls

@@ -1,15 +1,2 @@
 elasticsearch:
   templates:
-    - so/so-beats-template.json.jinja
-    - so/so-case-template.json.jinja
-    - so/so-common-template.json.jinja
-    - so/so-endgame-template.json.jinja
-    - so/so-firewall-template.json.jinja
-    - so/so-flow-template.json.jinja
-    - so/so-ids-template.json.jinja
-    - so/so-import-template.json.jinja
-    - so/so-osquery-template.json.jinja
-    - so/so-ossec-template.json.jinja
-    - so/so-strelka-template.json.jinja
-    - so/so-syslog-template.json.jinja
-    - so/so-zeek-template.json.jinja

pillar/elasticsearch/search.sls

@@ -1,15 +1,2 @@
 elasticsearch:
   templates:
-    - so/so-beats-template.json.jinja
-    - so/so-case-template.json.jinja
-    - so/so-common-template.json.jinja
-    - so/so-endgame-template.json.jinja
-    - so/so-firewall-template.json.jinja
-    - so/so-flow-template.json.jinja
-    - so/so-ids-template.json.jinja
-    - so/so-import-template.json.jinja
-    - so/so-osquery-template.json.jinja
-    - so/so-ossec-template.json.jinja
-    - so/so-strelka-template.json.jinja
-    - so/so-syslog-template.json.jinja
-    - so/so-zeek-template.json.jinja

salt/elasticsearch/base-template.json.jinja

@@ -0,0 +1 @@
+{{ TEMPLATE_CONFIG | tojson(true) }}

salt/elasticsearch/init.sls

@@ -41,7 +41,7 @@ include:
 {% set ROLES = salt['pillar.get']('elasticsearch:roles', {}) %}
 {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %}
 {% from 'elasticsearch/config.map.jinja' import ESCONFIG with context %}
-
+{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS without context %}
 
 vm.max_map_count:
   sysctl.present:
@@ -152,8 +152,6 @@ estemplatedir:
     - group: 939
     - makedirs: True
 
-
-
 esrolesdir:
   file.directory:
     - name: /opt/so/conf/elasticsearch/roles
@@ -198,7 +196,26 @@ esyml:
         ESCONFIG: {{ ESCONFIG }}
     - template: jinja
 
-#sync templates to /opt/so/conf/elasticsearch/templates
+escomponenttemplates:
+  file.recurse:
+    - name: /opt/so/conf/elasticsearch/templates/component
+    - source: salt://elasticsearch/templates/component
+    - user: 930
+    - group: 939
+
+# Auto-generate templates from defaults file
+{% for index, settings in ES_INDEX_SETTINGS.items() %}
+es_index_template_{{index}}:
+  file.managed:
+    - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json
+    - source: salt://elasticsearch/base-template.json.jinja
+    - defaults:
+      TEMPLATE_CONFIG: {{ settings.index_template }}
+    - template: jinja
+{% endfor %}
+
+{% if TEMPLATES %}
+# Sync custom templates to /opt/so/conf/elasticsearch/templates
 {% for TEMPLATE in TEMPLATES %}
 es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
   file.managed:
@@ -212,13 +229,7 @@ es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
     - user: 930
     - group: 939
 {% endfor %}
-
-escomponenttemplates:
-  file.recurse:
-    - name: /opt/so/conf/elasticsearch/templates/component
-    - source: salt://elasticsearch/templates/component
-    - user: 930
-    - group: 939
+{% endif %}
 
 esroles:
   file.recurse:
@@ -380,7 +391,6 @@ so-elasticsearch-pipelines:
       - docker_container: so-elasticsearch
       - file: so-elasticsearch-pipelines-script
 
-{% if TEMPLATES %}
 so-elasticsearch-templates:
   cmd.run:
     - name: /usr/sbin/so-elasticsearch-templates-load
@@ -389,7 +399,6 @@ so-elasticsearch-templates:
     - require:
       - docker_container: so-elasticsearch
       - file: es_sync_scripts
-{% endif %}
 
 so-elasticsearch-roles-load:
   cmd.run:

salt/elasticsearch/template.map.jinja

@@ -0,0 +1,7 @@
+{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %}
+{%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ESCONFIG.elasticsearch.index_settings, merge=True) %}
+{% for index, settings in ES_INDEX_SETTINGS.items() %}
+  {% if settings.index_sorting, False %}
+    {% do settings.index_template.template.settings.index.pop('sort') %}
+  {% endif %}
+{% endfor %}

salt/logstash/init.sls

@@ -199,9 +199,6 @@ so-logstash:
     {% for CONFIGFILE in PIPELINES[PL].config %}
       - file: ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}
     {% endfor %}
-  {% endfor %}
-  {% for TEMPLATE in TEMPLATES %}
-      - file: es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}
   {% endfor %}
     - require:
   {% if grains['role'] in ['so-manager', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %}