CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2026-01-28 10:53:34 +01:00
Code Issues Packages Projects Releases Wiki Activity

fixes addon integration map file

Browse Source 
Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com>
This commit is contained in:
reyesj2
2025-01-03 16:06:22 -06:00
parent ecf094f684
commit cdd4a1ff1f
6 changed files with 70 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
salt/elasticfleet/integration-defaults.map.jinja
View File

@@ -10,6 +10,44 @@
{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
{% set ADDON_INTEGRATION_DEFAULTS = {} %}
{# Some fleet integrations don't follow the standard naming convention #}
{% set WEIRD_INTEGRATIONS = {
'awsfirehose.logs': 'awsfirehose',
'cribl.logs': 'cribl',
'sentinel_one_cloud_funnel.logins': 'sentinel_one_cloud_funnel.login',
'azure_application_insights.app_insights': 'azure.app_insights',
'azure_application_insights.app_state': 'azure.app_state',
'azure_billing.billing': 'azure.billing',
'azure_functions.metrics': 'azure.function',
'azure_metrics.compute_vm_scaleset': 'azure.compute_vm_scaleset',
'azure_metrics.compute_vm': 'azure.compute_vm',
'azure_metrics.container_instance': 'azure.container_instance',
'azure_metrics.container_registry': 'azure.container_registry',
'azure_metrics.container_service': 'azure.container_service',
'azure_metrics.database_account': 'azure.database_account',
'azure_metrics.monitor': 'azure.monitor',
'azure_metrics.storage_account': 'azure.storage_account',
'azure_openai.metrics': 'azure.open_ai',
'beat.state': 'beats.stack_monitoring.state',
'beat.stats': 'beats.stack_monitoring.stats',
'enterprisesearch.health': 'enterprisesearch.stack_monitoring.health',
'enterprisesearch.stats': 'enterprisesearch.stack_monitoring.stats',
'kibana.cluster_actions': 'kibana.stack_monitoring.cluster_actions',
'kibana.cluster_rules': 'kibana.stack_monitoring.cluster_rules',
'kibana.node_actions': 'kibana.stack_monitoring.node_actions',
'kibana.node_rules': 'kibana.stack_monitoring.node_rules',
'kibana.stats': 'kibana.stack_monitoring.stats',
'kibana.status': 'kibana.stack_monitoring.status',
'logstash.node_cel': 'logstash.stack_monitoring.node',
'logstash.node_stats': 'logstash.stack_monitoring.node_stats',
'synthetics.browser': 'synthetics-browser',
'synthetics.browser_network': 'synthetics-browser.network',
'synthetics.browser_screenshot': 'synthetics-browser.screenshot',
'synthetics.http': 'synthetics-http',
'synthetics.icmp': 'synthetics-icmp',
'synthetics.tcp': 'synthetics-tcp'
} %}
{% for pkg in ADDON_PACKAGE_COMPONENTS %}
{% if pkg.name in CORE_ESFLEET_PACKAGES %}
{# skip core integrations #}
@@ -17,22 +55,36 @@
{# generate defaults for each integration #}
{% if pkg.es_index_patterns is defined and pkg.es_index_patterns is not none %}
{% for pattern in pkg.es_index_patterns %}
{% set integration_key = "so-logs-" ~ pkg.name ~ "_x_" ~ pattern.title %}
{% set integration_defaults = {
{% if "metrics-" in pattern.name %}
{% set integration_type = "metrics-" %}
{% elif "logs-" in pattern.name %}
{% set integration_type = "logs-" %}
{% else %}
{% set integration_type = "" %}
{% endif %}
{% set component_name = pkg.name ~ "." ~ pattern.title %}
{# fix weirdly named components #}
{% if component_name in WEIRD_INTEGRATIONS %}
{% set component_name = WEIRD_INTEGRATIONS[component_name] %}
{% endif %}
{% set integration_key = "so-" ~ integration_type ~ component_name %}
{# Default integration settings #}
{% set integration_defaults = {
"index_sorting": false,
"index_template": {
"composed_of": ["logs-" ~ pkg.name ~ "." ~ pattern.title ~ "@package", "logs-" ~ pkg.name ~ "." ~ pattern.title ~ "@custom", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
"composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
"data_stream": {
"hidden": false,
"allow_custom_routing": false
"allow_custom_routing": false,
"hidden": false
},
"ignore_missing_component_templates": ["logs-" ~ pkg.name ~ "." ~ pattern.title ~ "@custom"],
"ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
"index_patterns": [pattern.name],
"priority": 501,
"template": {
"settings": {
"index": {
"lifecycle": {"name": "so-logs-" ~ pkg.name ~ "." ~ pattern.title ~ "-logs"},
"lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
"number_of_replicas": 0
}
}

2
salt/elastic-fleet-package-registry/tools/so-elastic-fleet-optional-integrations-load → salt/elasticfleet/tools/sbin/so-elastic-fleet-optional-integrations-load
View File

@@ -78,7 +78,7 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then
if [ $PENDING_UPDATE ]; then
# Run bulk install of packages
# elastic_fleet_bulk_package_install $BULK_INSTALL_PACKAGE_LIST
elastic_fleet_bulk_package_install $BULK_INSTALL_PACKAGE_LIST
# Write out file for generating index/component/ilm templates
latest_installed_package_list=$(elastic_fleet_installed_packages)

184
salt/elasticsearch/defaults.yaml
View File

@@ -3297,190 +3297,6 @@ elasticsearch:
index:
mode: time_series
number_of_replicas: 0
so-metrics-nginx_x_stubstatus:
index_sorting: false
index_template:
composed_of:
- metrics-nginx.stubstatus@package
- metrics-nginx.stubstatus@custom
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates:
- metrics-nginx.stubstatus@custom
index_patterns:
- metrics-nginx.stubstatus-*
priority: 501
template:
settings:
index:
lifecycle:
name: so-metrics-nginx.stubstatus-logs
number_of_replicas: 0
policy:
phases:
cold:
actions:
set_priority:
priority: 0
min_age: 60d
delete:
actions:
delete: {}
min_age: 365d
hot:
actions:
rollover:
max_age: 30d
max_primary_shard_size: 50gb
set_priority:
priority: 100
min_age: 0ms
warm:
actions:
set_priority:
priority: 50
min_age: 30d
so-metrics-vsphere_x_datastore:
index_sorting: false
index_template:
composed_of:
- metrics-vsphere.datastore@package
- metrics-vsphere.datastore@custom
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates:
- metrics-vsphere.datastore@custom
index_patterns:
- metrics-vsphere.datastore-*
priority: 501
template:
settings:
index:
lifecycle:
name: so-metrics-vsphere.datastore-logs
number_of_replicas: 0
policy:
phases:
cold:
actions:
set_priority:
priority: 0
min_age: 60d
delete:
actions:
delete: {}
min_age: 365d
hot:
actions:
rollover:
max_age: 30d
max_primary_shard_size: 50gb
set_priority:
priority: 100
min_age: 0ms
warm:
actions:
set_priority:
priority: 50
min_age: 30d
so-metrics-vsphere_x_host:
index_sorting: false
index_template:
composed_of:
- metrics-vsphere.host@package
- metrics-vsphere.host@custom
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates:
- metrics-vsphere.host@custom
index_patterns:
- metrics-vsphere.host-*
priority: 501
template:
settings:
index:
lifecycle:
name: so-metrics-vsphere.host-logs
number_of_replicas: 0
policy:
phases:
cold:
actions:
set_priority:
priority: 0
min_age: 60d
delete:
actions:
delete: {}
min_age: 365d
hot:
actions:
rollover:
max_age: 30d
max_primary_shard_size: 50gb
set_priority:
priority: 100
min_age: 0ms
warm:
actions:
set_priority:
priority: 50
min_age: 30d
so-metrics-vsphere_x_virtualmachine:
index_sorting: false
index_template:
composed_of:
- metrics-vsphere.virtualmachine@package
- metrics-vsphere.virtualmachine@custom
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates:
- metrics-vsphere.virtualmachine@custom
index_patterns:
- metrics-vsphere.virtualmachine-*
priority: 501
template:
settings:
index:
lifecycle:
name: so-metrics-vsphere.virtualmachine-logs
number_of_replicas: 0
policy:
phases:
cold:
actions:
set_priority:
priority: 0
min_age: 60d
delete:
actions:
delete: {}
min_age: 365d
hot:
actions:
rollover:
max_age: 30d
max_primary_shard_size: 50gb
set_priority:
priority: 100
min_age: 0ms
warm:
actions:
set_priority:
priority: 50
min_age: 30d
so-redis:
index_sorting: false
index_template:

11
salt/elasticsearch/enabled.sls
View File

@@ -151,7 +151,7 @@ es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
{% endfor %}
{% endif %}
{% if GLOBALS.role in GLOBALS.manager_roles %}
{% if GLOBALS.role in GLOBALS.manager_roles %}
so-es-cluster-settings:
cmd.run:
- name: /usr/sbin/so-elasticsearch-cluster-settings
@@ -160,7 +160,7 @@ so-es-cluster-settings:
- require:
- docker_container: so-elasticsearch
- file: elasticsearch_sbin_jinja
{% endif %}
{% endif %}
so-elasticsearch-ilm-policy-load:
cmd.run:
@@ -172,6 +172,13 @@ so-elasticsearch-ilm-policy-load:
- onchanges:
- file: so-elasticsearch-ilm-policy-load-script
configure-addon-fleet-integrations:
cmd.run:
- name: /usr/sbin/so-elastic-fleet-optional-integrations-load
- cwd: /opt/so
- require:
- docker_container: so-elasticsearch
so-elasticsearch-templates-reload:
file.absent:
- name: /opt/so/state/estemplates.txt

110
salt/elasticsearch/integration-templates.map.jinja
View File

@@ -1,110 +0,0 @@
{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
https://securityonion.net/license; you may not use this file except in compliance with the
Elastic License 2.0. #}
{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
{% set packages = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
{% set INTEGRATION_INDEX_SETTINGS = {} %}
{% set default_settings = {
'index_sorting': false,
'index_template': {
'data_stream': {
'allow_custom_routing': false,
'hidden': false
},
'priority': 501,
'template': {
'settings': {
'index': {
'number_of_replicas': 0
}
}
}
},
'policy': {
'phases': {
'cold': {
'actions': {
'set_priority': {
'priority': 0
}
},
'min_age': '60d'
},
'delete': {
'actions': {
'delete': {}
},
'min_age': '365d'
},
'hot': {
'actions': {
'rollover':{
'max_age': '30d',
'max_primary_shard_size': '50gb'
},
'set_priority': {
'priority': 100
}
},
'min_age': '0ms'
},
'warm': {
'actions': {
'set_priority': {
'priority': 50
}
},
'min_age': '30d'
}
}
}
} %}
{# Create template for each package component from elasticfleet/defaults.yaml #}
{% for package in packages %}
{% for pkg_name, components in package.items() %}
{% if components is not none %}
{% for component in components %}
{% set component_dot = component.replace('_x_', '.') %}
{% set template_name = 'so-logs-' ~ component %}
{% set template = {
'index_sorting': default_settings.index_sorting,
'index_template': {
'composed_of': [
'logs-' ~ component_dot ~ '@package',
'logs-' ~ component_dot ~ '@custom',
'so-fleet-_globals-1',
'so-fleet_agent_id_verification-1'
],
'data_stream': default_settings.index_template.data_stream,
'ignore_missing_component_templates': [
'logs-' ~ component_dot ~ '@custom'
],
'index_patterns': [
'logs-' ~ component_dot ~ '-*'
],
'priority': default_settings.index_template.priority,
'template': {
'settings': {
'index': {
'lifecycle': {
'name': 'so-logs-' ~ component_dot ~ '-logs'
},
'number_of_replicas': default_settings.index_template.template.settings.index.number_of_replicas
}
}
}
},
'policy': default_settings.policy
} %}
{% do INTEGRATION_INDEX_SETTINGS.update({template_name: template}) %}
{% endfor %}
{% endif %}
{% endfor %}
{% endfor %}

2
salt/elasticsearch/template.map.jinja
View File

@@ -15,7 +15,7 @@
{% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %}
{# start generation of integration default index_settings #}
{% if salt['file.file_exists']('/opt/so/state/estemplates.txt') %}
{% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') %}
{% from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %}
{% for index, settings in ADDON_INTEGRATION_DEFAULTS.items() %}
{% do ES_INDEX_SETTINGS_ORIG.update({index: settings}) %}