Initial template refactor

salt/elasticsearch/base-template.json.jinja

@@ -0,0 +1 @@
+{{ TEMPLATE_CONFIG | tojson(true) }}

salt/elasticsearch/defaults.yaml

@@ -62,7 +62,85 @@ elasticsearch:
       org:
         elasticsearch:
           deprecation: ERROR
-
+  index_settings:
-
+    so-beats:
-
+      warm: 1
-
+      close: 1
+      delete: 900
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - so-beats-*
+        template:
+          mappings:
+            dynamic_templates:
+              - strings_as_keyword:
+                  mapping:
+                    ignore_above: 1024
+                    type: keyword
+                  match_mapping_type: string
+            date_detection: false
+          settings:
+            index:
+              mapping:
+                total_fields:
+                  limit: 3000
+              sort:
+                field: "@timestamp"
+                order: desc
+              refresh_interval: 30s
+              number_of_shards: 1
+              number_of_replicas: 0
+        composed_of: 
+          - agent-mappings
+          - dtc-agent-mappings
+          - base-mappings
+          - dtc-base-mappings
+          - client-mappings
+          - cloud-mappings
+          - container-mappings
+          - data_stream-mappings
+          - destination-mappings
+          - dll-mappings
+          - dns-mappings
+          - dtc-dns-mappings
+          - ecs-mappings
+          - dtc-ecs-mappings
+          - error-mappings
+          - event-mappings
+          - dtc-event-mappings
+          - file-mappings
+          - dtc-file-mappings
+          - group-mappings
+          - host-mappings
+          - dtc-host-mappings
+          - http-mappings
+          - dtc-http-mappings
+          - log-mappings
+          - network-mappings
+          - dtc-network-mappings
+          - observer-mappings
+          - dtc-observer-mappings
+          - orchestrator-mappings
+          - organization-mappings
+          - package-mappings
+          - process-mappings
+          - dtc-process-mappings
+          - registry-mappings
+          - related-mappings
+          - rule-mappings
+          - dtc-rule-mappings
+          - server-mappings
+          - service-mappings
+          - dtc-service-mappings
+          - source-mappings
+          - threat-mappings
+          - tls-mappings
+          - tracing-mappings
+          - url-mappings
+          - user_agent-mappings
+          - dtc-user_agent-mappings
+          - vulnerability-mappings
+          - common-settings
+          - common-dynamic-mappings
+          - winlog-mappings

salt/elasticsearch/init.sls

@@ -37,11 +37,10 @@ include:
   {% set ismanager = True %} {# Solely for the sake of running so-catrust #}
 {% endif %}
 
-{% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
 {% set ROLES = salt['pillar.get']('elasticsearch:roles', {}) %}
 {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %}
 {% from 'elasticsearch/config.map.jinja' import ESCONFIG with context %}
-
+{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS without context %}
 
 vm.max_map_count:
   sysctl.present:
@@ -152,8 +151,6 @@ estemplatedir:
     - group: 939
     - makedirs: True
 
-
-
 esrolesdir:
   file.directory:
     - name: /opt/so/conf/elasticsearch/roles
@@ -198,21 +195,6 @@ esyml:
         ESCONFIG: {{ ESCONFIG }}
     - template: jinja
 
-#sync templates to /opt/so/conf/elasticsearch/templates
-{% for TEMPLATE in TEMPLATES %}
-es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
-  file.managed:
-    - source: salt://elasticsearch/templates/index/{{TEMPLATE}}
-    {% if 'jinja' in TEMPLATE.split('.')[-1] %}
-    - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}}
-    - template: jinja
-    {% else %}
-    - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1]}}
-    {% endif %}
-    - user: 930
-    - group: 939
-{% endfor %}
-
 escomponenttemplates:
   file.recurse:
     - name: /opt/so/conf/elasticsearch/templates/component
@@ -220,6 +202,16 @@ escomponenttemplates:
     - user: 930
     - group: 939
 
+{% for index, settings in ES_INDEX_SETTINGS.items() %}
+es_index_template_{{index}}:
+  file.managed:
+    - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json
+    - source: salt://elasticsearch/base-template.json.jinja
+    - defaults:
+      TEMPLATE_CONFIG: {{ settings.index_template }}
+    - template: jinja
+{% endfor %}
+
 esroles:
   file.recurse:
     - source: salt://elasticsearch/roles/
@@ -380,7 +372,6 @@ so-elasticsearch-pipelines:
       - docker_container: so-elasticsearch
       - file: so-elasticsearch-pipelines-script
 
-{% if TEMPLATES %}
 so-elasticsearch-templates:
   cmd.run:
     - name: /usr/sbin/so-elasticsearch-templates-load
@@ -389,7 +380,6 @@ so-elasticsearch-templates:
     - require:
       - docker_container: so-elasticsearch
       - file: es_sync_scripts
-{% endif %}
 
 so-elasticsearch-roles-load:
   cmd.run: