From 9cba9d9ae05c44a1b9b0a7ccbacf13b45cafcc9d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 17 Aug 2023 15:00:01 -0400 Subject: [PATCH 1/3] allow to override number_of_replicas from one place in soc ui --- salt/elasticsearch/defaults.yaml | 6 ++++++ salt/elasticsearch/soc_elasticsearch.yaml | 11 ++++++++++ salt/elasticsearch/template.map.jinja | 25 +++++++++++++++++++---- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 579197040..5cb027fd2 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -58,6 +58,12 @@ elasticsearch: elasticsearch: deprecation: ERROR index_settings: + global_overrides: + index_template: + template: + settings: + index: + number_of_replicas: default_placeholder so-logs: index_sorting: False index_template: diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 889e9f6a4..bed6939e1 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -47,6 +47,16 @@ elasticsearch: global: True helpLink: elasticsearch.html index_settings: + global_overrides: + index_template: + template: + settings: + index: + number_of_replicas: + description: Number of replicas required for all indicies. Multiple replicas protects against data loss, but also increases storage costs. This setting will be applied to all indicies. + forcedType: int + global: True + helpLink: elasticsearch.html so-logs: &indexSettings index_sorting: description: Sorts the index by event time, at the cost of additional processing resource consumption. @@ -64,6 +74,7 @@ elasticsearch: index: number_of_replicas: description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs. + forcedType: int global: True helpLink: elasticsearch.html mapping: diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index 5fe0ed303..f92aa4c8f 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -1,11 +1,28 @@ -{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS with context %} -{%- set ES_INDEX_SETTINGS_ORIG = salt['pillar.get']('elasticsearch:index_settings', default=ELASTICSEARCHDEFAULTS.elasticsearch.index_settings, merge=True) %} -{% set ES_INDEX_SETTINGS = {} %} +{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %} +{% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %} + +{% set PILLAR_GLOBAL_OVERRIDES = {} %} +{% if salt['pillar.get']('elasticsearch:index_settings') is defined %} +{% set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings') %} +{% if ES_INDEX_PILLAR.global_overrides is defined %} +{% set PILLAR_GLOBAL_OVERRIDES = ES_INDEX_PILLAR.pop('global_overrides') %} +{% endif %} +{% endif %} + +{% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %} + +{% set ES_INDEX_SETTINGS_GLOBAL_OVERRIDES = {} %} {% for index, settings in ES_INDEX_SETTINGS_ORIG.items() %} +{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ELASTICSEARCHDEFAULTS.elasticsearch.index_settings[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %} {% if settings.index_template is defined %} {% if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %} {% do settings.index_template.template.settings.index.pop('sort') %} {% endif %} {% endif %} - {% do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_ORIG[index]}) %} +{% endfor %} + +{% set ES_INDEX_SETTINGS = {} %} +{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %} +{% for index in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.keys() %} + {% do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %} {% endfor %} From 4ac95447ebf38904f3669c10271cbfcaf8eb9795 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 17 Aug 2023 16:15:27 -0400 Subject: [PATCH 2/3] pop sort settings if index_sorting is false --- salt/elasticsearch/template.map.jinja | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index f92aa4c8f..f5a124a9a 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -12,17 +12,17 @@ {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %} {% set ES_INDEX_SETTINGS_GLOBAL_OVERRIDES = {} %} -{% for index, settings in ES_INDEX_SETTINGS_ORIG.items() %} -{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ELASTICSEARCHDEFAULTS.elasticsearch.index_settings[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %} - {% if settings.index_template is defined %} - {% if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %} - {% do settings.index_template.template.settings.index.pop('sort') %} - {% endif %} - {% endif %} +{% for index in ES_INDEX_SETTINGS_ORIG.keys() %} +{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ELASTICSEARCHDEFAULTS.elasticsearch.index_settings[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %} {% endfor %} {% set ES_INDEX_SETTINGS = {} %} {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %} -{% for index in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.keys() %} - {% do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %} +{% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %} +{% if settings.index_template is defined %} +{% if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %} +{% do settings.index_template.template.settings.index.pop('sort') %} +{% endif %} +{% endif %} +{% do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %} {% endfor %} From 222352b4b3c116adb5038eb7482eeddd6ea5e383 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 17 Aug 2023 17:26:35 -0400 Subject: [PATCH 3/3] fix typo --- salt/elasticsearch/soc_elasticsearch.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index bed6939e1..f269ec014 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -53,7 +53,7 @@ elasticsearch: settings: index: number_of_replicas: - description: Number of replicas required for all indicies. Multiple replicas protects against data loss, but also increases storage costs. This setting will be applied to all indicies. + description: Number of replicas required for all indices. Multiple replicas protects against data loss, but also increases storage costs. This setting will be applied to all indices. forcedType: int global: True helpLink: elasticsearch.html