CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-08 10:12:53 +01:00
Code Issues Packages Projects Releases Wiki Activity

changes for 'generic' integrations with no compoent templates assigned. Default to using the logs-filestream.generic@package componet template

Browse Source
This commit is contained in:
reyesj2
2025-07-08 15:23:46 -05:00
parent 4bf2c931e9
commit d4f0cbcb67
3 changed files with 81 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

132
salt/elasticfleet/integration-defaults.map.jinja
View File

@@ -4,6 +4,7 @@
{% import_json '/opt/so/state/esfleet_package_components.json' as ADDON_PACKAGE_COMPONENTS %} {% import_json '/opt/so/state/esfleet_package_components.json' as ADDON_PACKAGE_COMPONENTS %}
{% import_json '/opt/so/state/esfleet_package_components.json' as INSTALLED_COMPONENT_TEMPLATES %}
{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} {% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %} {% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
@@ -62,70 +63,89 @@
{% else %} {% else %}
{% set integration_type = "" %} {% set integration_type = "" %}
{% endif %} {% endif %}
{% set component_name = pkg.name ~ "." ~ pattern.title %} {% set component_name = pkg.name ~ "." ~ pattern.title %}
{# fix weirdly named components #} {% set index_pattern = pattern.name %}
{% if component_name in WEIRD_INTEGRATIONS %}
{% set component_name = WEIRD_INTEGRATIONS[component_name] %} {# fix weirdly named components #}
{% endif %} {% if component_name in WEIRD_INTEGRATIONS %}
{% set component_name = WEIRD_INTEGRATIONS[component_name] %}
{% endif %}
{# create duplicate of component_name, so we can split generics from @custom component templates in the index template below and overwrite the default @package when needed
eg. having to replace unifiedlogs.generic@package with filestream.generic@package, but keep the ability to customize unifiedlogs.generic@custom and its ILM policy #}
{% set custom_component_name = component_name %}
{# duplicate integration_type to assist with sometimes needing to overwrite component templates with 'logs-filestream.generic@package' (there is no metrics-filestream.generic@package) #}
{% set generic_integration_type = integration_type %}
{# component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #} {# component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
{% set component_name_x = component_name.replace(".","_x_") %} {% set component_name_x = component_name.replace(".","_x_") %}
{# pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #} {# pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
{% set integration_key = "so-" ~ integration_type ~ component_name_x %} {% set integration_key = "so-" ~ integration_type ~ component_name_x %}
{% if ".generic" in component_name and component_name not in INSTALLED_COMPONENT_TEMPLATES %}
{# these generic templates by default are directed to index_pattern of 'logs-generic-*', overwrite that here to point to eg gcp_pubsub.generic-* #}
{% set index_pattern = integration_type ~ component_name ~ "-*" %}
{# includes use of .generic component template, but it doesn't exist in installed component templates. Redirect it to filestream.generic@package #}
{% set component_name = "filestream.generic" %}
{% set generic_integration_type = "logs-" %}
{% endif %}
{# Default integration settings #} {# Default integration settings #}
{% set integration_defaults = { {% set integration_defaults = {
"index_sorting": false, "index_sorting": false,
"index_template": { "index_template": {
"composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"], "composed_of": [generic_integration_type ~ component_name ~ "@package", integration_type ~ custom_component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
"data_stream": { "data_stream": {
"allow_custom_routing": false, "allow_custom_routing": false,
"hidden": false "hidden": false
}, },
"ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"], "ignore_missing_component_templates": [integration_type ~ custom_component_name ~ "@custom"],
"index_patterns": [pattern.name], "index_patterns": [index_pattern],
"priority": 501, "priority": 501,
"template": { "template": {
"settings": { "settings": {
"index": { "index": {
"lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"}, "lifecycle": {"name": "so-" ~ integration_type ~ custom_component_name ~ "-logs"},
"number_of_replicas": 0 "number_of_replicas": 0
} }
} }
} }
}, },
"policy": { "policy": {
"phases": { "phases": {
"cold": { "cold": {
"actions": { "actions": {
"set_priority": {"priority": 0} "set_priority": {"priority": 0}
}, },
"min_age": "60d" "min_age": "60d"
},
"delete": {
"actions": {
"delete": {}
},
"min_age": "365d"
},
"hot": {
"actions": {
"rollover": {
"max_age": "30d",
"max_primary_shard_size": "50gb"
},
"set_priority": {"priority": 100}
}, },
"delete": { "min_age": "0ms"
"actions": { },
"delete": {} "warm": {
}, "actions": {
"min_age": "365d" "set_priority": {"priority": 50}
}, },
"hot": { "min_age": "30d"
"actions": { }
"rollover": { }
"max_age": "30d", }
"max_primary_shard_size": "50gb" } %}
},
"set_priority": {"priority": 100}
},
"min_age": "0ms"
},
"warm": {
"actions": {
"set_priority": {"priority": 50}
},
"min_age": "30d"
}
}
}
} %}
{% do ADDON_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %} {% do ADDON_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
{% endfor %} {% endfor %}
{% endif %} {% endif %}

4
salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load
View File

@@ -19,6 +19,7 @@ BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json
BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json
BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json
PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
COMPONENT_TEMPLATES=/opt/so/state/esfleet_component_templates.txt
PENDING_UPDATE=false PENDING_UPDATE=false
@@ -155,6 +156,9 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then
# Write out file for generating index/component/ilm templates # Write out file for generating index/component/ilm templates
latest_installed_package_list=$(elastic_fleet_installed_packages) latest_installed_package_list=$(elastic_fleet_installed_packages)
echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS
# Refresh installed component template list
latest_component_templates_list=$(so-elasticsearch-query _component_template | jq '.component_templates[] | .name' | jq -s '.')
echo $latest_component_templates_list > $COMPONENT_TEMPLATES
else else
# This is the installation of add-on integrations and upgrade of existing integrations. Exiting without error, next highstate will attempt to re-run. # This is the installation of add-on integrations and upgrade of existing integrations. Exiting without error, next highstate will attempt to re-run.

2
salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load
View File

@@ -136,7 +136,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then
TEMPLATE=${i::-14} TEMPLATE=${i::-14}
COMPONENT_PATTERN=${TEMPLATE:3} COMPONENT_PATTERN=${TEMPLATE:3}
MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery") MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery")
if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ logs-http_endpoint\.generic|logs-winlog\.winlog ]]; then if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ *\.generic|logs-winlog\.winlog ]]; then
load_failures=$((load_failures+1)) load_failures=$((load_failures+1))
echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures" echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures"
else else