From f68e3e47a1f7d0d29096ec22fc39bec3f2d2b699 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 16 Jun 2026 09:19:10 -0500 Subject: [PATCH 1/5] remove pillar merge --- salt/elasticsearch/template.map.jinja | 2 +- .../tools/sbin_jinja/so-elasticsearch-dlm-apply | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index 7acb3eb87..15481ee4c 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -5,7 +5,7 @@ {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %} {% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %} -{% set DATA_RETENTION_METHOD = salt['pillar.get']('elasticsearch:data_retention_method', ELASTICSEARCHDEFAULTS.elasticsearch.get('data_retention_method', 'ILM')) %} +{% set DATA_RETENTION_METHOD = salt['pillar.get']('elasticsearch:data_retention_method', ELASTICSEARCHDEFAULTS.elasticsearch.data_retention_method) %} {% set PILLAR_GLOBAL_OVERRIDES = {} %} {% set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings', {}) %} diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply index af761973c..e25e07223 100644 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply @@ -6,9 +6,8 @@ . /usr/sbin/so-common -{%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %} - -{%- set DATA_RETENTION_METHOD = salt['pillar.get']('elasticsearch:data_retention_method', ELASTICSEARCHDEFAULTS.elasticsearch.get('data_retention_method', 'ILM')) %} +{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} +{%- set DATA_RETENTION_METHOD = ELASTICSEARCHMERGED.data_retention_method %} ELASTICSEARCH_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR:-/opt/so/conf/elasticsearch/templates}" TEMPLATE_DIRS=( From a769d4c68094c8b19fcb455f0ece69953ba9d7fc Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 16 Jun 2026 09:32:37 -0500 Subject: [PATCH 2/5] another unneeded default --- salt/elasticsearch/template.map.jinja | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index 15481ee4c..72be1ec58 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -4,8 +4,11 @@ Elastic License 2.0. #} {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %} +{# ELASTICSEARCHMERGED only used here to collect data_retention_method. This file intentionally works with ELASTICSEARCHDEFAULTS #} +{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} + {% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %} -{% set DATA_RETENTION_METHOD = salt['pillar.get']('elasticsearch:data_retention_method', ELASTICSEARCHDEFAULTS.elasticsearch.data_retention_method) %} +{% set DATA_RETENTION_METHOD = ELASTICSEARCHMERGED.data_retention_method %} {% set PILLAR_GLOBAL_OVERRIDES = {} %} {% set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings', {}) %} From 4a6c6752236a5b3cf98f7717f8a9e4b3d7a59169 Mon Sep 17 00:00:00 2001 From: Jorge Reyes <94730068+reyesj2@users.noreply.github.com> Date: Tue, 16 Jun 2026 10:33:11 -0500 Subject: [PATCH 3/5] skip kibana backport if the template doesn't exist --- salt/manager/tools/sbin/soup | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 0016cc00e..668d7d1de 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -784,9 +784,8 @@ pin_elasticsearch_data_retention_method() { # Reference: https://github.com/elastic/kibana/issues/263048 kibana_backport_streams_index_template() { local current_template updated_template - current_template=$(so-elasticsearch-query "_index_template/.kibana_streams" --retry 3 --retry-delay 5 --fail) - if [[ -z "$current_template" ]]; then + if ! current_template=$(so-elasticsearch-query "_index_template/.kibana_streams" --retry 3 --retry-delay 5 --fail); then echo "Index template .kibana_streams does not exist, skipping backport." return 0 fi From 4456bde1c820c2c5cbda1054e1398c010fed375b Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 16 Jun 2026 10:45:53 -0500 Subject: [PATCH 4/5] check if template exists without --fail flag --- salt/manager/tools/sbin/soup | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 668d7d1de..12be85eaf 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -783,10 +783,16 @@ pin_elasticsearch_data_retention_method() { # # Reference: https://github.com/elastic/kibana/issues/263048 kibana_backport_streams_index_template() { - local current_template updated_template + local current_template updated_template current_template_exists + + current_template_exists=$(so-elasticsearch-query "_index_template/.kibana_streams" -o /dev/null -w "%{http_code}") + if [[ "$current_template_exists" == "404" ]]; then + echo "Index template .kibana_streams does not exist, skipping backport." + return 0 + fi if ! current_template=$(so-elasticsearch-query "_index_template/.kibana_streams" --retry 3 --retry-delay 5 --fail); then - echo "Index template .kibana_streams does not exist, skipping backport." + echo "Unable to retrieve .kibana_streams index template, skipping backport." return 0 fi From 3daed551df0a3e435be6f98b341630f808e2d88d Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 16 Jun 2026 11:17:04 -0500 Subject: [PATCH 5/5] use --fail flag without set -x, since elasticsearch can return a 404 on the template lookup --- salt/manager/tools/sbin/soup | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 12be85eaf..96313aea4 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -783,18 +783,14 @@ pin_elasticsearch_data_retention_method() { # # Reference: https://github.com/elastic/kibana/issues/263048 kibana_backport_streams_index_template() { - local current_template updated_template current_template_exists + local current_template updated_template - current_template_exists=$(so-elasticsearch-query "_index_template/.kibana_streams" -o /dev/null -w "%{http_code}") - if [[ "$current_template_exists" == "404" ]]; then + set +e + if ! current_template=$(so-elasticsearch-query "_index_template/.kibana_streams" --retry 3 --retry-delay 5 --fail); then echo "Index template .kibana_streams does not exist, skipping backport." return 0 fi - - if ! current_template=$(so-elasticsearch-query "_index_template/.kibana_streams" --retry 3 --retry-delay 5 --fail); then - echo "Unable to retrieve .kibana_streams index template, skipping backport." - return 0 - fi + set -e updated_template=$(jq '.index_templates[0].index_template | .template.settings += {"index.auto_expand_replicas": "0-1"} | del(.created_date_millis, .modified_date_millis)' <<< "$current_template")