diff --git a/salt/elasticsearch/base-template.json.jinja b/salt/elasticsearch/base-template.json.jinja new file mode 100644 index 000000000..02aed1b49 --- /dev/null +++ b/salt/elasticsearch/base-template.json.jinja @@ -0,0 +1 @@ +{{ TEMPLATE_CONFIG | tojson(true) }} diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index bfeeadb20..5011da46a 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -62,7 +62,85 @@ elasticsearch: org: elasticsearch: deprecation: ERROR - - - - + index_settings: + so-beats: + warm: 1 + close: 1 + delete: 900 + index_sorting: False + index_template: + index_patterns: + - so-beats-* + template: + mappings: + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string + date_detection: false + settings: + index: + mapping: + total_fields: + limit: 3000 + sort: + field: "@timestamp" + order: desc + refresh_interval: 30s + number_of_shards: 1 + number_of_replicas: 0 + composed_of: + - agent-mappings + - dtc-agent-mappings + - base-mappings + - dtc-base-mappings + - client-mappings + - cloud-mappings + - container-mappings + - data_stream-mappings + - destination-mappings + - dll-mappings + - dns-mappings + - dtc-dns-mappings + - ecs-mappings + - dtc-ecs-mappings + - error-mappings + - event-mappings + - dtc-event-mappings + - file-mappings + - dtc-file-mappings + - group-mappings + - host-mappings + - dtc-host-mappings + - http-mappings + - dtc-http-mappings + - log-mappings + - network-mappings + - dtc-network-mappings + - observer-mappings + - dtc-observer-mappings + - orchestrator-mappings + - organization-mappings + - package-mappings + - process-mappings + - dtc-process-mappings + - registry-mappings + - related-mappings + - rule-mappings + - dtc-rule-mappings + - server-mappings + - service-mappings + - dtc-service-mappings + - source-mappings + - threat-mappings + - tls-mappings + - tracing-mappings + - url-mappings + - user_agent-mappings + - dtc-user_agent-mappings + - vulnerability-mappings + - common-settings + - common-dynamic-mappings + - winlog-mappings diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index aa3c6cbd6..fcafb3da1 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -37,11 +37,10 @@ include: {% set ismanager = True %} {# Solely for the sake of running so-catrust #} {% endif %} -{% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} {% set ROLES = salt['pillar.get']('elasticsearch:roles', {}) %} {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} {% from 'elasticsearch/config.map.jinja' import ESCONFIG with context %} - +{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS without context %} vm.max_map_count: sysctl.present: @@ -152,8 +151,6 @@ estemplatedir: - group: 939 - makedirs: True - - esrolesdir: file.directory: - name: /opt/so/conf/elasticsearch/roles @@ -198,21 +195,6 @@ esyml: ESCONFIG: {{ ESCONFIG }} - template: jinja -#sync templates to /opt/so/conf/elasticsearch/templates -{% for TEMPLATE in TEMPLATES %} -es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}: - file.managed: - - source: salt://elasticsearch/templates/index/{{TEMPLATE}} - {% if 'jinja' in TEMPLATE.split('.')[-1] %} - - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}} - - template: jinja - {% else %} - - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1]}} - {% endif %} - - user: 930 - - group: 939 -{% endfor %} - escomponenttemplates: file.recurse: - name: /opt/so/conf/elasticsearch/templates/component @@ -220,6 +202,16 @@ escomponenttemplates: - user: 930 - group: 939 +{% for index, settings in ES_INDEX_SETTINGS.items() %} +es_index_template_{{index}}: + file.managed: + - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json + - source: salt://elasticsearch/base-template.json.jinja + - defaults: + TEMPLATE_CONFIG: {{ settings.index_template }} + - template: jinja +{% endfor %} + esroles: file.recurse: - source: salt://elasticsearch/roles/ @@ -380,7 +372,6 @@ so-elasticsearch-pipelines: - docker_container: so-elasticsearch - file: so-elasticsearch-pipelines-script -{% if TEMPLATES %} so-elasticsearch-templates: cmd.run: - name: /usr/sbin/so-elasticsearch-templates-load @@ -389,7 +380,6 @@ so-elasticsearch-templates: - require: - docker_container: so-elasticsearch - file: es_sync_scripts -{% endif %} so-elasticsearch-roles-load: cmd.run: