From cdd4a1ff1fb6b6fc2c7b95651593746713d8b795 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 3 Jan 2025 16:06:22 -0600 Subject: [PATCH] fixes addon integration map file Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- .../integration-defaults.map.jinja | 66 ++++++- ...o-elastic-fleet-optional-integrations-load | 2 +- salt/elasticsearch/defaults.yaml | 184 ------------------ salt/elasticsearch/enabled.sls | 11 +- .../integration-templates.map.jinja | 110 ----------- salt/elasticsearch/template.map.jinja | 2 +- 6 files changed, 70 insertions(+), 305 deletions(-) rename salt/{elastic-fleet-package-registry/tools => elasticfleet/tools/sbin}/so-elastic-fleet-optional-integrations-load (98%) delete mode 100644 salt/elasticsearch/integration-templates.map.jinja diff --git a/salt/elasticfleet/integration-defaults.map.jinja b/salt/elasticfleet/integration-defaults.map.jinja index 9977856c4..0de400b26 100644 --- a/salt/elasticfleet/integration-defaults.map.jinja +++ b/salt/elasticfleet/integration-defaults.map.jinja @@ -10,6 +10,44 @@ {% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %} {% set ADDON_INTEGRATION_DEFAULTS = {} %} +{# Some fleet integrations don't follow the standard naming convention #} +{% set WEIRD_INTEGRATIONS = { + 'awsfirehose.logs': 'awsfirehose', + 'cribl.logs': 'cribl', + 'sentinel_one_cloud_funnel.logins': 'sentinel_one_cloud_funnel.login', + 'azure_application_insights.app_insights': 'azure.app_insights', + 'azure_application_insights.app_state': 'azure.app_state', + 'azure_billing.billing': 'azure.billing', + 'azure_functions.metrics': 'azure.function', + 'azure_metrics.compute_vm_scaleset': 'azure.compute_vm_scaleset', + 'azure_metrics.compute_vm': 'azure.compute_vm', + 'azure_metrics.container_instance': 'azure.container_instance', + 'azure_metrics.container_registry': 'azure.container_registry', + 'azure_metrics.container_service': 'azure.container_service', + 'azure_metrics.database_account': 'azure.database_account', + 'azure_metrics.monitor': 'azure.monitor', + 'azure_metrics.storage_account': 'azure.storage_account', + 'azure_openai.metrics': 'azure.open_ai', + 'beat.state': 'beats.stack_monitoring.state', + 'beat.stats': 'beats.stack_monitoring.stats', + 'enterprisesearch.health': 'enterprisesearch.stack_monitoring.health', + 'enterprisesearch.stats': 'enterprisesearch.stack_monitoring.stats', + 'kibana.cluster_actions': 'kibana.stack_monitoring.cluster_actions', + 'kibana.cluster_rules': 'kibana.stack_monitoring.cluster_rules', + 'kibana.node_actions': 'kibana.stack_monitoring.node_actions', + 'kibana.node_rules': 'kibana.stack_monitoring.node_rules', + 'kibana.stats': 'kibana.stack_monitoring.stats', + 'kibana.status': 'kibana.stack_monitoring.status', + 'logstash.node_cel': 'logstash.stack_monitoring.node', + 'logstash.node_stats': 'logstash.stack_monitoring.node_stats', + 'synthetics.browser': 'synthetics-browser', + 'synthetics.browser_network': 'synthetics-browser.network', + 'synthetics.browser_screenshot': 'synthetics-browser.screenshot', + 'synthetics.http': 'synthetics-http', + 'synthetics.icmp': 'synthetics-icmp', + 'synthetics.tcp': 'synthetics-tcp' + } %} + {% for pkg in ADDON_PACKAGE_COMPONENTS %} {% if pkg.name in CORE_ESFLEET_PACKAGES %} {# skip core integrations #} @@ -17,22 +55,36 @@ {# generate defaults for each integration #} {% if pkg.es_index_patterns is defined and pkg.es_index_patterns is not none %} {% for pattern in pkg.es_index_patterns %} -{% set integration_key = "so-logs-" ~ pkg.name ~ "_x_" ~ pattern.title %} -{% set integration_defaults = { +{% if "metrics-" in pattern.name %} +{% set integration_type = "metrics-" %} +{% elif "logs-" in pattern.name %} +{% set integration_type = "logs-" %} +{% else %} +{% set integration_type = "" %} +{% endif %} +{% set component_name = pkg.name ~ "." ~ pattern.title %} +{# fix weirdly named components #} +{% if component_name in WEIRD_INTEGRATIONS %} +{% set component_name = WEIRD_INTEGRATIONS[component_name] %} +{% endif %} +{% set integration_key = "so-" ~ integration_type ~ component_name %} + +{# Default integration settings #} +{% set integration_defaults = { "index_sorting": false, "index_template": { - "composed_of": ["logs-" ~ pkg.name ~ "." ~ pattern.title ~ "@package", "logs-" ~ pkg.name ~ "." ~ pattern.title ~ "@custom", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"], + "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"], "data_stream": { - "hidden": false, - "allow_custom_routing": false + "allow_custom_routing": false, + "hidden": false }, - "ignore_missing_component_templates": ["logs-" ~ pkg.name ~ "." ~ pattern.title ~ "@custom"], + "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"], "index_patterns": [pattern.name], "priority": 501, "template": { "settings": { "index": { - "lifecycle": {"name": "so-logs-" ~ pkg.name ~ "." ~ pattern.title ~ "-logs"}, + "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"}, "number_of_replicas": 0 } } diff --git a/salt/elastic-fleet-package-registry/tools/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin/so-elastic-fleet-optional-integrations-load similarity index 98% rename from salt/elastic-fleet-package-registry/tools/so-elastic-fleet-optional-integrations-load rename to salt/elasticfleet/tools/sbin/so-elastic-fleet-optional-integrations-load index d94b006ad..5fa14c5fc 100644 --- a/salt/elastic-fleet-package-registry/tools/so-elastic-fleet-optional-integrations-load +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-optional-integrations-load @@ -78,7 +78,7 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then if [ $PENDING_UPDATE ]; then # Run bulk install of packages - # elastic_fleet_bulk_package_install $BULK_INSTALL_PACKAGE_LIST + elastic_fleet_bulk_package_install $BULK_INSTALL_PACKAGE_LIST # Write out file for generating index/component/ilm templates latest_installed_package_list=$(elastic_fleet_installed_packages) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index e7a9a286c..32d9c431e 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -3297,190 +3297,6 @@ elasticsearch: index: mode: time_series number_of_replicas: 0 - so-metrics-nginx_x_stubstatus: - index_sorting: false - index_template: - composed_of: - - metrics-nginx.stubstatus@package - - metrics-nginx.stubstatus@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - metrics-nginx.stubstatus@custom - index_patterns: - - metrics-nginx.stubstatus-* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-metrics-nginx.stubstatus-logs - number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-vsphere_x_datastore: - index_sorting: false - index_template: - composed_of: - - metrics-vsphere.datastore@package - - metrics-vsphere.datastore@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - metrics-vsphere.datastore@custom - index_patterns: - - metrics-vsphere.datastore-* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-metrics-vsphere.datastore-logs - number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-vsphere_x_host: - index_sorting: false - index_template: - composed_of: - - metrics-vsphere.host@package - - metrics-vsphere.host@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - metrics-vsphere.host@custom - index_patterns: - - metrics-vsphere.host-* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-metrics-vsphere.host-logs - number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-vsphere_x_virtualmachine: - index_sorting: false - index_template: - composed_of: - - metrics-vsphere.virtualmachine@package - - metrics-vsphere.virtualmachine@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - metrics-vsphere.virtualmachine@custom - index_patterns: - - metrics-vsphere.virtualmachine-* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-metrics-vsphere.virtualmachine-logs - number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d so-redis: index_sorting: false index_template: diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index 48280c506..fb3f877df 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -151,7 +151,7 @@ es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}: {% endfor %} {% endif %} -{% if GLOBALS.role in GLOBALS.manager_roles %} +{% if GLOBALS.role in GLOBALS.manager_roles %} so-es-cluster-settings: cmd.run: - name: /usr/sbin/so-elasticsearch-cluster-settings @@ -160,7 +160,7 @@ so-es-cluster-settings: - require: - docker_container: so-elasticsearch - file: elasticsearch_sbin_jinja -{% endif %} +{% endif %} so-elasticsearch-ilm-policy-load: cmd.run: @@ -172,6 +172,13 @@ so-elasticsearch-ilm-policy-load: - onchanges: - file: so-elasticsearch-ilm-policy-load-script +configure-addon-fleet-integrations: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-optional-integrations-load + - cwd: /opt/so + - require: + - docker_container: so-elasticsearch + so-elasticsearch-templates-reload: file.absent: - name: /opt/so/state/estemplates.txt diff --git a/salt/elasticsearch/integration-templates.map.jinja b/salt/elasticsearch/integration-templates.map.jinja deleted file mode 100644 index 59a9222c5..000000000 --- a/salt/elasticsearch/integration-templates.map.jinja +++ /dev/null @@ -1,110 +0,0 @@ -{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one - or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at - https://securityonion.net/license; you may not use this file except in compliance with the - Elastic License 2.0. #} - -{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} -{% set packages = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %} -{% set INTEGRATION_INDEX_SETTINGS = {} %} - - -{% set default_settings = { - 'index_sorting': false, - 'index_template': { - 'data_stream': { - 'allow_custom_routing': false, - 'hidden': false - }, - 'priority': 501, - 'template': { - 'settings': { - 'index': { - 'number_of_replicas': 0 - } - } - } - }, - 'policy': { - 'phases': { - 'cold': { - 'actions': { - 'set_priority': { - 'priority': 0 - } - }, - 'min_age': '60d' - }, - 'delete': { - 'actions': { - 'delete': {} - }, - 'min_age': '365d' - }, - 'hot': { - 'actions': { - 'rollover':{ - 'max_age': '30d', - 'max_primary_shard_size': '50gb' - }, - 'set_priority': { - 'priority': 100 - } - }, - 'min_age': '0ms' - }, - 'warm': { - 'actions': { - 'set_priority': { - 'priority': 50 - } - }, - 'min_age': '30d' - } - } - } -} %} - -{# Create template for each package component from elasticfleet/defaults.yaml #} -{% for package in packages %} - {% for pkg_name, components in package.items() %} - {% if components is not none %} - {% for component in components %} - {% set component_dot = component.replace('_x_', '.') %} - {% set template_name = 'so-logs-' ~ component %} - - {% set template = { - 'index_sorting': default_settings.index_sorting, - 'index_template': { - 'composed_of': [ - 'logs-' ~ component_dot ~ '@package', - 'logs-' ~ component_dot ~ '@custom', - 'so-fleet-_globals-1', - 'so-fleet_agent_id_verification-1' - ], - 'data_stream': default_settings.index_template.data_stream, - 'ignore_missing_component_templates': [ - 'logs-' ~ component_dot ~ '@custom' - ], - 'index_patterns': [ - 'logs-' ~ component_dot ~ '-*' - ], - 'priority': default_settings.index_template.priority, - 'template': { - 'settings': { - 'index': { - 'lifecycle': { - 'name': 'so-logs-' ~ component_dot ~ '-logs' - }, - 'number_of_replicas': default_settings.index_template.template.settings.index.number_of_replicas - } - } - } - }, - 'policy': default_settings.policy - } %} - - {% do INTEGRATION_INDEX_SETTINGS.update({template_name: template}) %} - {% endfor %} - {% endif %} - {% endfor %} -{% endfor %} \ No newline at end of file diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index c53349f18..c1ff2cb24 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -15,7 +15,7 @@ {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %} {# start generation of integration default index_settings #} -{% if salt['file.file_exists']('/opt/so/state/estemplates.txt') %} +{% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') %} {% from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %} {% for index, settings in ADDON_INTEGRATION_DEFAULTS.items() %} {% do ES_INDEX_SETTINGS_ORIG.update({index: settings}) %}