diff --git a/salt/elasticfleet/integration-defaults.map.jinja b/salt/elasticfleet/integration-defaults.map.jinja index 5449e791e..62e37b229 100644 --- a/salt/elasticfleet/integration-defaults.map.jinja +++ b/salt/elasticfleet/integration-defaults.map.jinja @@ -4,6 +4,7 @@ {% import_json '/opt/so/state/esfleet_package_components.json' as ADDON_PACKAGE_COMPONENTS %} +{% import_json '/opt/so/state/esfleet_package_components.json' as INSTALLED_COMPONENT_TEMPLATES %} {% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} {% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %} @@ -62,70 +63,89 @@ {% else %} {% set integration_type = "" %} {% endif %} -{% set component_name = pkg.name ~ "." ~ pattern.title %} -{# fix weirdly named components #} -{% if component_name in WEIRD_INTEGRATIONS %} -{% set component_name = WEIRD_INTEGRATIONS[component_name] %} -{% endif %} +{% set component_name = pkg.name ~ "." ~ pattern.title %} +{% set index_pattern = pattern.name %} + +{# fix weirdly named components #} +{% if component_name in WEIRD_INTEGRATIONS %} +{% set component_name = WEIRD_INTEGRATIONS[component_name] %} +{% endif %} + +{# create duplicate of component_name, so we can split generics from @custom component templates in the index template below and overwrite the default @package when needed + eg. having to replace unifiedlogs.generic@package with filestream.generic@package, but keep the ability to customize unifiedlogs.generic@custom and its ILM policy #} +{% set custom_component_name = component_name %} + +{# duplicate integration_type to assist with sometimes needing to overwrite component templates with 'logs-filestream.generic@package' (there is no metrics-filestream.generic@package) #} +{% set generic_integration_type = integration_type %} + {# component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #} {% set component_name_x = component_name.replace(".","_x_") %} {# pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #} {% set integration_key = "so-" ~ integration_type ~ component_name_x %} +{% if ".generic" in component_name and component_name not in INSTALLED_COMPONENT_TEMPLATES %} +{# these generic templates by default are directed to index_pattern of 'logs-generic-*', overwrite that here to point to eg gcp_pubsub.generic-* #} +{% set index_pattern = integration_type ~ component_name ~ "-*" %} +{# includes use of .generic component template, but it doesn't exist in installed component templates. Redirect it to filestream.generic@package #} +{% set component_name = "filestream.generic" %} +{% set generic_integration_type = "logs-" %} +{% endif %} + {# Default integration settings #} {% set integration_defaults = { - "index_sorting": false, - "index_template": { - "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"], - "data_stream": { - "allow_custom_routing": false, - "hidden": false - }, - "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"], - "index_patterns": [pattern.name], - "priority": 501, - "template": { - "settings": { - "index": { - "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"}, - "number_of_replicas": 0 - } - } - } - }, - "policy": { - "phases": { - "cold": { - "actions": { - "set_priority": {"priority": 0} - }, - "min_age": "60d" + "index_sorting": false, + "index_template": { + "composed_of": [generic_integration_type ~ component_name ~ "@package", integration_type ~ custom_component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"], + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "ignore_missing_component_templates": [integration_type ~ custom_component_name ~ "@custom"], + "index_patterns": [index_pattern], + "priority": 501, + "template": { + "settings": { + "index": { + "lifecycle": {"name": "so-" ~ integration_type ~ custom_component_name ~ "-logs"}, + "number_of_replicas": 0 + } + } + } + }, + "policy": { + "phases": { + "cold": { + "actions": { + "set_priority": {"priority": 0} + }, + "min_age": "60d" + }, + "delete": { + "actions": { + "delete": {} + }, + "min_age": "365d" + }, + "hot": { + "actions": { + "rollover": { + "max_age": "30d", + "max_primary_shard_size": "50gb" + }, + "set_priority": {"priority": 100} }, - "delete": { - "actions": { - "delete": {} - }, - "min_age": "365d" - }, - "hot": { - "actions": { - "rollover": { - "max_age": "30d", - "max_primary_shard_size": "50gb" - }, - "set_priority": {"priority": 100} - }, - "min_age": "0ms" - }, - "warm": { - "actions": { - "set_priority": {"priority": 50} - }, - "min_age": "30d" - } - } - } - } %} + "min_age": "0ms" + }, + "warm": { + "actions": { + "set_priority": {"priority": 50} + }, + "min_age": "30d" + } + } + } + } %} + {% do ADDON_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %} {% endfor %} {% endif %} diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load index 26d775e82..df01608f5 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load @@ -19,6 +19,7 @@ BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json +COMPONENT_TEMPLATES=/opt/so/state/esfleet_component_templates.txt PENDING_UPDATE=false @@ -155,6 +156,9 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then # Write out file for generating index/component/ilm templates latest_installed_package_list=$(elastic_fleet_installed_packages) echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS + # Refresh installed component template list + latest_component_templates_list=$(so-elasticsearch-query _component_template | jq '.component_templates[] | .name' | jq -s '.') + echo $latest_component_templates_list > $COMPONENT_TEMPLATES else # This is the installation of add-on integrations and upgrade of existing integrations. Exiting without error, next highstate will attempt to re-run. diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load index 758f601eb..ba5767255 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -136,7 +136,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then TEMPLATE=${i::-14} COMPONENT_PATTERN=${TEMPLATE:3} MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery") - if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ logs-http_endpoint\.generic|logs-winlog\.winlog ]]; then + if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ *\.generic|logs-winlog\.winlog ]]; then load_failures=$((load_failures+1)) echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures" else