fixes addon integration map file

Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com>

salt/elasticfleet/integration-defaults.map.jinja

@@ -10,6 +10,44 @@
 {% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
 {% set ADDON_INTEGRATION_DEFAULTS = {} %}
 
+{# Some fleet integrations don't follow the standard naming convention #}
+{% set WEIRD_INTEGRATIONS = {
+    'awsfirehose.logs': 'awsfirehose',
+    'cribl.logs': 'cribl',
+    'sentinel_one_cloud_funnel.logins': 'sentinel_one_cloud_funnel.login',
+    'azure_application_insights.app_insights': 'azure.app_insights',
+    'azure_application_insights.app_state': 'azure.app_state',
+    'azure_billing.billing': 'azure.billing',
+    'azure_functions.metrics': 'azure.function',
+    'azure_metrics.compute_vm_scaleset': 'azure.compute_vm_scaleset',
+    'azure_metrics.compute_vm': 'azure.compute_vm',
+    'azure_metrics.container_instance': 'azure.container_instance',
+    'azure_metrics.container_registry': 'azure.container_registry',
+    'azure_metrics.container_service': 'azure.container_service',
+    'azure_metrics.database_account': 'azure.database_account',
+    'azure_metrics.monitor': 'azure.monitor',
+    'azure_metrics.storage_account': 'azure.storage_account',
+    'azure_openai.metrics': 'azure.open_ai',
+    'beat.state': 'beats.stack_monitoring.state',
+    'beat.stats': 'beats.stack_monitoring.stats',
+    'enterprisesearch.health': 'enterprisesearch.stack_monitoring.health',
+    'enterprisesearch.stats': 'enterprisesearch.stack_monitoring.stats',
+    'kibana.cluster_actions': 'kibana.stack_monitoring.cluster_actions',
+    'kibana.cluster_rules': 'kibana.stack_monitoring.cluster_rules',
+    'kibana.node_actions': 'kibana.stack_monitoring.node_actions',
+    'kibana.node_rules': 'kibana.stack_monitoring.node_rules',
+    'kibana.stats': 'kibana.stack_monitoring.stats',
+    'kibana.status': 'kibana.stack_monitoring.status',
+    'logstash.node_cel': 'logstash.stack_monitoring.node',
+    'logstash.node_stats': 'logstash.stack_monitoring.node_stats',
+    'synthetics.browser': 'synthetics-browser',
+    'synthetics.browser_network': 'synthetics-browser.network',
+    'synthetics.browser_screenshot': 'synthetics-browser.screenshot',
+    'synthetics.http': 'synthetics-http',
+    'synthetics.icmp': 'synthetics-icmp',
+    'synthetics.tcp': 'synthetics-tcp'
+   } %}
+
 {% for pkg in ADDON_PACKAGE_COMPONENTS %}
 {%   if pkg.name in CORE_ESFLEET_PACKAGES %}
 {#     skip core integrations #}
@@ -17,22 +55,36 @@
 {#     generate defaults for each integration #}
 {%     if pkg.es_index_patterns is defined and pkg.es_index_patterns is not none %}
 {%       for pattern in pkg.es_index_patterns %}
-{%         set integration_key = "so-logs-" ~ pkg.name ~ "_x_" ~ pattern.title %}
+{%         if "metrics-" in pattern.name %}
-{%         set integration_defaults = {
+{%           set integration_type = "metrics-" %}
+{%         elif "logs-" in pattern.name %}
+{%           set integration_type = "logs-" %}
+{%         else %}
+{%           set integration_type = "" %}
+{%         endif %}
+{%           set component_name = pkg.name ~ "." ~ pattern.title %}
+{#           fix weirdly named components #}
+{%           if component_name in WEIRD_INTEGRATIONS %}
+{%             set component_name = WEIRD_INTEGRATIONS[component_name] %}
+{%           endif %}
+{%           set integration_key = "so-" ~ integration_type ~ component_name %}
+
+{#           Default integration settings #}
+{%           set integration_defaults = {
               "index_sorting": false,
               "index_template": {
-                  "composed_of": ["logs-" ~ pkg.name ~ "." ~ pattern.title ~ "@package", "logs-" ~ pkg.name ~ "." ~ pattern.title ~ "@custom", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
+                  "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
                   "data_stream": {
-                      "hidden": false,
+                      "allow_custom_routing": false,
-                      "allow_custom_routing": false
+                      "hidden": false
                   },
-                  "ignore_missing_component_templates": ["logs-" ~ pkg.name ~ "." ~ pattern.title ~ "@custom"],
+                  "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
                   "index_patterns": [pattern.name],
                   "priority": 501,
                   "template": {
                       "settings": {
                           "index": {
-                              "lifecycle": {"name": "so-logs-" ~ pkg.name ~ "." ~ pattern.title ~ "-logs"},
+                              "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
                               "number_of_replicas": 0
                           }
                       }

salt/elasticfleet/tools/sbin/so-elastic-fleet-optional-integrations-load

@@ -78,7 +78,7 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
 
         if [ $PENDING_UPDATE ]; then
             # Run bulk install of packages
-            # elastic_fleet_bulk_package_install $BULK_INSTALL_PACKAGE_LIST
+            elastic_fleet_bulk_package_install $BULK_INSTALL_PACKAGE_LIST
 
             # Write out file for generating index/component/ilm templates
             latest_installed_package_list=$(elastic_fleet_installed_packages)

salt/elasticsearch/defaults.yaml

@@ -3297,190 +3297,6 @@ elasticsearch:
             index:
               mode: time_series
               number_of_replicas: 0
-    so-metrics-nginx_x_stubstatus:
-      index_sorting: false
-      index_template:
-        composed_of:
-        - metrics-nginx.stubstatus@package
-        - metrics-nginx.stubstatus@custom
-        - so-fleet_globals-1
-        - so-fleet_agent_id_verification-1
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
-        ignore_missing_component_templates:
-        - metrics-nginx.stubstatus@custom
-        index_patterns:
-        - metrics-nginx.stubstatus-*
-        priority: 501
-        template:
-          settings:
-            index:
-              lifecycle:
-                name: so-metrics-nginx.stubstatus-logs
-              number_of_replicas: 0
-      policy:
-        phases:
-          cold:
-            actions:
-              set_priority:
-                priority: 0
-            min_age: 60d
-          delete:
-            actions:
-              delete: {}
-            min_age: 365d
-          hot:
-            actions:
-              rollover:
-                max_age: 30d
-                max_primary_shard_size: 50gb
-              set_priority:
-                priority: 100
-            min_age: 0ms
-          warm:
-            actions:
-              set_priority:
-                priority: 50
-            min_age: 30d
-    so-metrics-vsphere_x_datastore:
-      index_sorting: false
-      index_template:
-        composed_of:
-        - metrics-vsphere.datastore@package
-        - metrics-vsphere.datastore@custom
-        - so-fleet_globals-1
-        - so-fleet_agent_id_verification-1
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
-        ignore_missing_component_templates:
-        - metrics-vsphere.datastore@custom
-        index_patterns:
-        - metrics-vsphere.datastore-*
-        priority: 501
-        template:
-          settings:
-            index:
-              lifecycle:
-                name: so-metrics-vsphere.datastore-logs
-              number_of_replicas: 0
-      policy:
-        phases:
-          cold:
-            actions:
-              set_priority:
-                priority: 0
-            min_age: 60d
-          delete:
-            actions:
-              delete: {}
-            min_age: 365d
-          hot:
-            actions:
-              rollover:
-                max_age: 30d
-                max_primary_shard_size: 50gb
-              set_priority:
-                priority: 100
-            min_age: 0ms
-          warm:
-            actions:
-              set_priority:
-                priority: 50
-            min_age: 30d
-    so-metrics-vsphere_x_host:
-      index_sorting: false
-      index_template:
-        composed_of:
-        - metrics-vsphere.host@package
-        - metrics-vsphere.host@custom
-        - so-fleet_globals-1
-        - so-fleet_agent_id_verification-1
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
-        ignore_missing_component_templates:
-        - metrics-vsphere.host@custom
-        index_patterns:
-        - metrics-vsphere.host-*
-        priority: 501
-        template:
-          settings:
-            index:
-              lifecycle:
-                name: so-metrics-vsphere.host-logs
-              number_of_replicas: 0
-      policy:
-        phases:
-          cold:
-            actions:
-              set_priority:
-                priority: 0
-            min_age: 60d
-          delete:
-            actions:
-              delete: {}
-            min_age: 365d
-          hot:
-            actions:
-              rollover:
-                max_age: 30d
-                max_primary_shard_size: 50gb
-              set_priority:
-                priority: 100
-            min_age: 0ms
-          warm:
-            actions:
-              set_priority:
-                priority: 50
-            min_age: 30d
-    so-metrics-vsphere_x_virtualmachine:
-      index_sorting: false
-      index_template:
-        composed_of:
-        - metrics-vsphere.virtualmachine@package
-        - metrics-vsphere.virtualmachine@custom
-        - so-fleet_globals-1
-        - so-fleet_agent_id_verification-1
-        data_stream:
-          allow_custom_routing: false
-          hidden: false
-        ignore_missing_component_templates:
-        - metrics-vsphere.virtualmachine@custom
-        index_patterns:
-        - metrics-vsphere.virtualmachine-*
-        priority: 501
-        template:
-          settings:
-            index:
-              lifecycle:
-                name: so-metrics-vsphere.virtualmachine-logs
-              number_of_replicas: 0
-      policy:
-        phases:
-          cold:
-            actions:
-              set_priority:
-                priority: 0
-            min_age: 60d
-          delete:
-            actions:
-              delete: {}
-            min_age: 365d
-          hot:
-            actions:
-              rollover:
-                max_age: 30d
-                max_primary_shard_size: 50gb
-              set_priority:
-                priority: 100
-            min_age: 0ms
-          warm:
-            actions:
-              set_priority:
-                priority: 50
-            min_age: 30d
     so-redis:
       index_sorting: false
       index_template:

salt/elasticsearch/enabled.sls

@@ -151,7 +151,7 @@ es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
 {%       endfor %}
 {%     endif %}
 
-{% if GLOBALS.role in GLOBALS.manager_roles %}
+{%     if GLOBALS.role in GLOBALS.manager_roles %}
 so-es-cluster-settings:
   cmd.run:
     - name: /usr/sbin/so-elasticsearch-cluster-settings
@@ -160,7 +160,7 @@ so-es-cluster-settings:
     - require:
       - docker_container: so-elasticsearch
       - file: elasticsearch_sbin_jinja
-{% endif %}
+{%     endif %}
 
 so-elasticsearch-ilm-policy-load:
   cmd.run:
@@ -172,6 +172,13 @@ so-elasticsearch-ilm-policy-load:
     - onchanges:
       - file: so-elasticsearch-ilm-policy-load-script
 
+configure-addon-fleet-integrations:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-optional-integrations-load
+    - cwd: /opt/so
+    - require:
+      - docker_container: so-elasticsearch
+
 so-elasticsearch-templates-reload:
   file.absent:
     - name: /opt/so/state/estemplates.txt

salt/elasticsearch/integration-templates.map.jinja

@@ -1,110 +0,0 @@
-{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-   https://securityonion.net/license; you may not use this file except in compliance with the
-   Elastic License 2.0. #}
-
-{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
-{% set packages = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
-{% set INTEGRATION_INDEX_SETTINGS = {} %}
-
-
-{% set default_settings = {
-    'index_sorting': false,
-    'index_template': {
-        'data_stream': {
-          'allow_custom_routing': false,
-          'hidden': false
-        },
-        'priority': 501,
-        'template': {
-            'settings': {
-                'index': {
-                    'number_of_replicas': 0
-                }
-            }
-        }
-    },
-    'policy': {
-        'phases': {
-          'cold': {
-            'actions': {
-              'set_priority': {
-                'priority': 0
-              }
-            },
-            'min_age': '60d'
-          },
-          'delete': {
-            'actions': {
-              'delete': {}
-            },
-            'min_age': '365d'
-          },
-          'hot': {
-            'actions': {
-              'rollover':{
-                'max_age': '30d',
-                'max_primary_shard_size': '50gb'
-              },
-              'set_priority': {
-                'priority': 100
-              }
-            },
-            'min_age': '0ms'
-          },
-          'warm': {
-            'actions': {
-              'set_priority': {
-                'priority': 50
-              }
-            },
-            'min_age': '30d'
-          }
-        }
-    }
-} %}
-
-{# Create template for each package component from elasticfleet/defaults.yaml #}
-{% for package in packages %}
-    {% for pkg_name, components in package.items() %}
-        {% if components is not none %}
-            {% for component in components %}
-                {% set component_dot = component.replace('_x_', '.') %}
-                {% set template_name = 'so-logs-' ~ component %}
-
-                {% set template = {
-                    'index_sorting': default_settings.index_sorting,
-                    'index_template': {
-                        'composed_of': [
-                            'logs-' ~ component_dot ~ '@package',
-                            'logs-' ~ component_dot ~ '@custom',
-                            'so-fleet-_globals-1',
-                            'so-fleet_agent_id_verification-1'
-                        ],
-                        'data_stream': default_settings.index_template.data_stream,
-                        'ignore_missing_component_templates': [
-                            'logs-' ~ component_dot ~  '@custom'
-                        ],
-                        'index_patterns': [
-                            'logs-' ~ component_dot ~ '-*'
-                        ],
-                        'priority': default_settings.index_template.priority,
-                        'template': {
-                            'settings': {
-                                'index': {
-                                    'lifecycle': {
-                                        'name': 'so-logs-' ~ component_dot ~ '-logs'
-                                    },
-                                    'number_of_replicas': default_settings.index_template.template.settings.index.number_of_replicas
-                                }
-                            }
-                        }
-                    },
-                    'policy': default_settings.policy
-                } %}
-
-                {% do INTEGRATION_INDEX_SETTINGS.update({template_name: template}) %}
-            {% endfor %}
-        {% endif %}
-    {% endfor %}
-{% endfor %}

salt/elasticsearch/template.map.jinja

@@ -15,7 +15,7 @@
 {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %}
 
 {# start generation of integration default index_settings #}
-{% if salt['file.file_exists']('/opt/so/state/estemplates.txt') %}
+{% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') %}
 {%   from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %}
 {%   for index, settings in ADDON_INTEGRATION_DEFAULTS.items() %}
 {%     do ES_INDEX_SETTINGS_ORIG.update({index: settings}) %}