CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2026-06-22 18:28:11 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3/dev' into soupmod

Browse Source
This commit is contained in:
Josh Patterson
2026-06-22 09:41:16 -04:00
parent 0a69833669 63a2e20698
commit c950ac7370
23 changed files with 1043 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
salt/elasticsearch/cluster.sls
+14 -1
View File
@@ -133,6 +133,18 @@ so-elasticsearch-templates:
- docker_container: so-elasticsearch
- file: elasticsearch_sbin_jinja
so-elasticsearch-dlm-apply:
cmd.run:
- name: /usr/sbin/so-elasticsearch-dlm-apply
- cwd: /opt/so
- require:
- docker_container: so-elasticsearch
- file: elasticsearch_sbin_jinja
- cmd: so-elasticsearch-templates
- retry:
attempts: 3
interval: 10
so-elasticsearch-pipelines:
cmd.run:
- name: /usr/sbin/so-elasticsearch-pipelines {{ GLOBALS.hostname }}
@@ -153,7 +165,8 @@ so-elasticsearch-roles-load:
{% set ap = "absent" %}
{% endif %}
{% if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %}
{% if ELASTICSEARCHMERGED.index_clean %}
{# Remove so-elasticsearch-indices-delete script when using DLM #}
{% if ELASTICSEARCHMERGED.index_clean and ELASTICSEARCHMERGED.data_retention_method == "ILM" %}
{% set ap = "present" %}
{% else %}
{% set ap = "absent" %}
salt/elasticsearch/defaults.yaml
+135 -7
View File
@@ -2,6 +2,7 @@ elasticsearch:
enabled: false
version: 9.3.3
index_clean: true
data_retention_method: DLM
vm:
max_map_count: 1048576
config:
@@ -63,6 +64,8 @@ elasticsearch:
verification_mode: none
index_settings:
global_overrides:
data_stream_lifecycle:
data_retention: 90d
index_template:
template:
settings:
@@ -143,6 +146,8 @@ elasticsearch:
order: desc
so-common:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- agent-mappings
@@ -304,6 +309,8 @@ elasticsearch:
number_of_shards: 1
so-assistant-chat:
index_sorting: false
data_stream_lifecycle:
data_retention: ""
index_template:
composed_of:
- assistant-chat-mappings
@@ -344,6 +351,8 @@ elasticsearch:
min_age: 0ms
so-assistant-session:
index_sorting: false
data_stream_lifecycle:
data_retention: ""
index_template:
composed_of:
- assistant-session-mappings
@@ -497,6 +506,8 @@ elasticsearch:
min_age: 30d
so-idh:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- agent-mappings
@@ -605,6 +616,8 @@ elasticsearch:
min_age: 30d
so-import:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- agent-mappings
@@ -787,6 +800,8 @@ elasticsearch:
min_age: 0ms
so-kismet:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- kismet-mappings
@@ -836,6 +851,8 @@ elasticsearch:
min_age: 30d
so-kratos:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- agent-mappings
@@ -904,6 +921,8 @@ elasticsearch:
min_age: 30d
so-hydra:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- agent-mappings
@@ -1049,6 +1068,8 @@ elasticsearch:
min_age: 0ms
so-logs:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- so-data-streams-mappings
@@ -1129,6 +1150,8 @@ elasticsearch:
min_age: 30d
so-logs-detections_x_alerts:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- so-data-streams-mappings
@@ -1192,6 +1215,8 @@ elasticsearch:
min_age: 30d
so-logs-elastic_agent:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- event-mappings
@@ -1307,6 +1332,8 @@ elasticsearch:
min_age: 30d
so-elastic-agent-monitor:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- event-mappings
@@ -1369,6 +1396,8 @@ elasticsearch:
min_age: 30d
so-logs-elastic_agent_x_apm_server:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- logs-elastic_agent.apm_server@package
@@ -1433,6 +1462,8 @@ elasticsearch:
min_age: 30d
so-logs-elastic_agent_x_auditbeat:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- logs-elastic_agent.auditbeat@package
@@ -1497,6 +1528,8 @@ elasticsearch:
min_age: 30d
so-logs-elastic_agent_x_cloudbeat:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- logs-elastic_agent.cloudbeat@package
@@ -1561,6 +1594,8 @@ elasticsearch:
min_age: 30d
so-logs-elastic_agent_x_endpoint_security:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- event-mappings
@@ -1620,6 +1655,8 @@ elasticsearch:
min_age: 30d
so-logs-elastic_agent_x_filebeat:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- event-mappings
@@ -1679,6 +1716,8 @@ elasticsearch:
min_age: 30d
so-logs-elastic_agent_x_fleet_server:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- event-mappings
@@ -1735,6 +1774,8 @@ elasticsearch:
min_age: 30d
so-logs-elastic_agent_x_heartbeat:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- logs-elastic_agent.heartbeat@package
@@ -1799,6 +1840,8 @@ elasticsearch:
min_age: 30d
so-logs-elastic_agent_x_metricbeat:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- event-mappings
@@ -1858,6 +1901,8 @@ elasticsearch:
min_age: 30d
so-logs-elastic_agent_x_osquerybeat:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- event-mappings
@@ -1917,6 +1962,8 @@ elasticsearch:
min_age: 30d
so-logs-elastic_agent_x_packetbeat:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- logs-elastic_agent.packetbeat@package
@@ -1981,6 +2028,8 @@ elasticsearch:
min_age: 30d
so-logs-elasticsearch_x_server:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- logs-elasticsearch.server@package
@@ -2045,10 +2094,13 @@ elasticsearch:
min_age: 30d
so-logs-endpoint_x_actions:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- .logs-endpoint.actions@package
- .logs-endpoint.actions@custom
- endpoint@custom
- event-mappings
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
@@ -2058,8 +2110,9 @@ elasticsearch:
hidden: false
ignore_missing_component_templates:
- .logs-endpoint.actions@custom
- endpoint@custom
index_patterns:
- logs-endpoint.actions-*
- .logs-endpoint.actions-*
priority: 501
template:
settings:
@@ -2104,10 +2157,13 @@ elasticsearch:
min_age: 30d
so-logs-endpoint_x_action_x_responses:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- .logs-endpoint.action.responses@package
- .logs-endpoint.action.responses@custom
- endpoint@custom
- event-mappings
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
@@ -2117,14 +2173,15 @@ elasticsearch:
hidden: false
ignore_missing_component_templates:
- .logs-endpoint.action.responses@custom
- endpoint@custom
index_patterns:
- logs-endpoint.action.responses-*
- .logs-endpoint.action.responses-*
priority: 501
template:
settings:
index:
lifecycle:
name: so-logs-endpoint.actions-logs
name: so-logs-endpoint.action.responses-logs
mapping:
total_fields:
limit: 5000
@@ -2163,6 +2220,8 @@ elasticsearch:
min_age: 30d
so-logs-endpoint_x_alerts:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- logs-endpoint.alerts@package
@@ -2222,6 +2281,8 @@ elasticsearch:
min_age: 30d
so-logs-endpoint_x_diagnostic_x_collection:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- .logs-endpoint.diagnostic.collection@package
@@ -2297,6 +2358,8 @@ elasticsearch:
min_age: 30d
so-logs-endpoint_x_events_x_api:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- logs-endpoint.events.api@package
@@ -2356,6 +2419,8 @@ elasticsearch:
min_age: 30d
so-logs-endpoint_x_events_x_file:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- logs-endpoint.events.file@package
@@ -2415,6 +2480,8 @@ elasticsearch:
min_age: 30d
so-logs-endpoint_x_events_x_library:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- logs-endpoint.events.library@package
@@ -2474,6 +2541,8 @@ elasticsearch:
min_age: 30d
so-logs-endpoint_x_events_x_network:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- logs-endpoint.events.network@package
@@ -2533,6 +2602,8 @@ elasticsearch:
min_age: 30d
so-logs-endpoint_x_events_x_process:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- logs-endpoint.events.process@package
@@ -2592,6 +2663,8 @@ elasticsearch:
min_age: 30d
so-logs-endpoint_x_events_x_registry:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- logs-endpoint.events.registry@package
@@ -2651,6 +2724,8 @@ elasticsearch:
min_age: 30d
so-logs-endpoint_x_events_x_security:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- logs-endpoint.events.security@package
@@ -2710,6 +2785,8 @@ elasticsearch:
min_age: 30d
so-logs-endpoint_x_heartbeat:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- .logs-endpoint.heartbeat@package
@@ -2769,6 +2846,8 @@ elasticsearch:
min_age: 30d
so-logs-http_endpoint_x_generic:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- logs-http_endpoint.generic@package
@@ -2817,6 +2896,8 @@ elasticsearch:
min_age: 30d
so-logs-httpjson_x_generic:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- logs-httpjson.generic@package
@@ -2882,6 +2963,8 @@ elasticsearch:
number_of_replicas: 0
so-logs-osquery-manager_x_action_x_responses:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
_meta:
managed: true
@@ -2953,6 +3036,8 @@ elasticsearch:
number_of_replicas: 0
so-logs-osquery-manager_x_result:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
_meta:
managed: true
@@ -3005,6 +3090,8 @@ elasticsearch:
min_age: 30d
so-logs-soc:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- agent-mappings
@@ -3113,6 +3200,8 @@ elasticsearch:
min_age: 30d
so-logs-system_x_application:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- event-mappings
@@ -3162,6 +3251,8 @@ elasticsearch:
min_age: 30d
so-logs-system_x_auth:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- event-mappings
@@ -3211,6 +3302,8 @@ elasticsearch:
min_age: 30d
so-logs-system_x_security:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- event-mappings
@@ -3260,6 +3353,8 @@ elasticsearch:
min_age: 30d
so-logs-system_x_syslog:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- event-mappings
@@ -3309,6 +3404,8 @@ elasticsearch:
min_age: 30d
so-logs-system_x_system:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- event-mappings
@@ -3358,6 +3455,8 @@ elasticsearch:
min_age: 30d
so-logs-windows_x_forwarded:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- logs-windows.forwarded@package
@@ -3405,6 +3504,8 @@ elasticsearch:
min_age: 30d
so-logs-windows_x_powershell:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- logs-windows.powershell@package
@@ -3452,6 +3553,8 @@ elasticsearch:
min_age: 30d
so-logs-windows_x_powershell_operational:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- logs-windows.powershell_operational@package
@@ -3499,6 +3602,8 @@ elasticsearch:
min_age: 30d
so-logs-windows_x_sysmon_operational:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- logs-windows.sysmon_operational@package
@@ -3546,6 +3651,8 @@ elasticsearch:
min_age: 30d
so-logs-winlog_x_winlog:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- logs-winlog.winlog@package
@@ -3594,6 +3701,8 @@ elasticsearch:
min_age: 30d
so-logstash:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- agent-mappings
@@ -3709,6 +3818,8 @@ elasticsearch:
min_age: 30d
so-metrics-endpoint_x_metadata:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- metrics-endpoint.metadata@package
@@ -3756,6 +3867,8 @@ elasticsearch:
min_age: 30d
so-metrics-endpoint_x_metrics:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- metrics-endpoint.metrics@package
@@ -3803,6 +3916,8 @@ elasticsearch:
min_age: 30d
so-metrics-endpoint_x_policy:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- metrics-endpoint.policy@package
@@ -3850,6 +3965,8 @@ elasticsearch:
min_age: 30d
so-metrics-fleet_server_x_agent_status:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- metrics@tsdb-settings
@@ -3874,6 +3991,8 @@ elasticsearch:
number_of_replicas: 0
so-metrics-fleet_server_x_agent_versions:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- metrics@tsdb-settings
@@ -3898,6 +4017,8 @@ elasticsearch:
number_of_replicas: 0
so-redis:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- agent-mappings
@@ -3958,13 +4079,10 @@ elasticsearch:
- vulnerability-mappings
- common-settings
- common-dynamic-mappings
- logs-redis.log@package
- logs-redis.log@custom
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates:
- logs-redis.log@custom
ignore_missing_component_templates: []
index_patterns:
- logs-redis.log*
priority: 501
@@ -4016,6 +4134,8 @@ elasticsearch:
min_age: 30d
so-strelka:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- agent-mappings
@@ -4133,6 +4253,8 @@ elasticsearch:
min_age: 30d
so-suricata:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- agent-mappings
@@ -4249,6 +4371,8 @@ elasticsearch:
min_age: 30d
so-suricata_x_alerts:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- agent-mappings
@@ -4365,6 +4489,8 @@ elasticsearch:
min_age: 30d
so-syslog:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- agent-mappings
@@ -4481,6 +4607,8 @@ elasticsearch:
min_age: 30d
so-zeek:
index_sorting: false
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- agent-mappings
salt/elasticsearch/soc_elasticsearch.yaml
+394 -61
View File
@@ -4,6 +4,13 @@ elasticsearch:
forcedType: bool
advanced: True
helpLink: elasticsearch
data_retention_method:
description: Method for data retention. Options are ILM or DLM. For single node deployments and most distributed grid users, DLM will be the recommended option for simplified management. Those with more complex use cases may prefer ILM. The latter allows for more granular control, but requires more management overhead.
options:
- ILM
- DLM
forcedType: string
global: True
version:
description: "This specifies the version of the following containers: so-elastic-fleet-package-registry, so-elastic-agent, so-elastic-fleet, so-kibana, so-logstash and so-elasticsearch. Modifying this value in the Elasticsearch defaults.yaml will result in catastrophic grid failure."
readonly: True
@@ -13,7 +20,7 @@ elasticsearch:
description: Specify the memory heap size in (m)egabytes for Elasticsearch.
helpLink: elasticsearch
index_clean:
description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, indices will only be deleted by the age defined in the ILM settings. This setting only applies to EVAL, STANDALONE, and HEAVY NODE installations. Other installations can only use ILM settings.
description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, data is retained by the configured lifecycle settings. This setting only applies to EVAL, STANDALONE, and HEAVY NODE installations. Other installations use lifecycle settings only.
forcedType: bool
helpLink: elasticsearch
vm:
@@ -139,6 +146,23 @@ elasticsearch:
custom010: *pipelines
index_settings:
global_overrides:
data_stream_lifecycle:
data_retention:
description: |
The retention period for all data streams. Retention does not define the period that the data will be removed, but the minimum time period they will be kept.
Use a number followed by a time unit, such as 7d. Leave blank for indefinite retention where supported.
Configured retention period also affects the frequency of rolling over data streams.
- If retention is less than or equal to 1 day, max_age will be 1 hour
- If retention is less than or equal to 14 days, max_age will be 1 day
- If retention is less than or equal to 90 days, max_age will be 7 days
- If retention is greater than 90 days, max_age will be 30 days
forcedType: string
allowedNodeTypes:
- heavynode
regex: ^$|^[0-9]{1,5}(?:d|h|m|s)$
regexFailureMessage: Must be blank or a number followed by d, h, m, or s, such as 7d.
index_template:
template:
settings:
@@ -311,13 +335,30 @@ elasticsearch:
forcedType: string
global: True
helpLink: elasticsearch
so-logs: &indexSettings
so-logs: &dataStreamSettings
index_sorting:
description: Sorts the index by event time, at the cost of additional processing resource consumption.
forcedType: bool
global: True
advanced: True
helpLink: elasticsearch
data_stream_lifecycle:
data_retention:
description: |
The retention period for this data stream. Retention does not define the period that the data will be removed, but the minimum time period it will be kept.
Use a number followed by a time unit, such as 7d. Leave blank for indefinite retention where supported.
Configured retention period also affects the frequency of rolling over this data stream.
- If retention is less than or equal to 1 day, max_age will be 1 hour
- If retention is less than or equal to 14 days, max_age will be 1 day
- If retention is less than or equal to 90 days, max_age will be 7 days
- If retention is greater than 90 days, max_age will be 30 days
forcedType: string
allowedNodeTypes:
- heavynode
regex: ^$|^[0-9]{1,5}(?:d|h|m|s)$
regexFailureMessage: Must be blank or a number followed by d, h, m, or s, such as 7d.
index_template:
index_patterns:
description: Patterns for matching multiple indices or tables.
@@ -335,6 +376,14 @@ elasticsearch:
global: True
advanced: True
helpLink: elasticsearch
auto_expand_replicas:
description: Automatically expand the number of replicas based on the number of data nodes in the cluster. This can help ensure high availability as the cluster scales up or down.
forcedType: string
regex: "^(0-[1-9]|1-[2-9]|2-[3-9]|3-[4-9]|4-[5-9]|5-[6-9]|6-[7-9]|7-[89]|8-9|[0-9]-all|false)$"
regexFailureMessage: Must be in the format of "x-y" where x is minimum number of replicas and y is maximum number of replicas, or "0-all" to specify a minimum of 0 and no maximum, or "false" to disable automatic replica expansion.
global: True
advanced: True
helpLink: elasticsearch
mapping:
total_fields:
limit:
@@ -596,65 +645,349 @@ elasticsearch:
global: True
advanced: True
helpLink: elasticsearch
so-logs-system_x_auth: *indexSettings
so-logs-system_x_syslog: *indexSettings
so-logs-system_x_system: *indexSettings
so-logs-system_x_application: *indexSettings
so-logs-system_x_security: *indexSettings
so-logs-windows_x_forwarded: *indexSettings
so-logs-windows_x_powershell: *indexSettings
so-logs-windows_x_powershell_operational: *indexSettings
so-logs-windows_x_sysmon_operational: *indexSettings
so-logs-winlog_x_winlog: *indexSettings
so-logs-detections_x_alerts: *indexSettings
so-logs-http_endpoint_x_generic: *indexSettings
so-logs-httpjson_x_generic: *indexSettings
so-logs-osquery-manager-actions: *indexSettings
so-logs-osquery-manager-action_x_responses: *indexSettings
so-logs-osquery-manager_x_action_x_responses: *indexSettings
so-logs-osquery-manager_x_result: *indexSettings
so-logs-elastic_agent_x_apm_server: *indexSettings
so-logs-elastic_agent_x_auditbeat: *indexSettings
so-logs-elastic_agent_x_cloudbeat: *indexSettings
so-logs-elastic_agent_x_endpoint_security: *indexSettings
so-logs-endpoint_x_alerts: *indexSettings
so-logs-endpoint_x_events_x_api: *indexSettings
so-logs-endpoint_x_events_x_file: *indexSettings
so-logs-endpoint_x_events_x_library: *indexSettings
so-logs-endpoint_x_events_x_network: *indexSettings
so-logs-endpoint_x_events_x_process: *indexSettings
so-logs-endpoint_x_events_x_registry: *indexSettings
so-logs-endpoint_x_events_x_security: *indexSettings
so-logs-elastic_agent_x_filebeat: *indexSettings
so-logs-elastic_agent_x_fleet_server: *indexSettings
so-logs-elastic_agent_x_heartbeat: *indexSettings
so-logs-elastic_agent: *indexSettings
so-logs-elastic_agent_x_metricbeat: *indexSettings
so-logs-elastic_agent_x_osquerybeat: *indexSettings
so-logs-elastic_agent_x_packetbeat: *indexSettings
so-logs-elasticsearch_x_server: *indexSettings
so-metrics-endpoint_x_metadata: *indexSettings
so-metrics-endpoint_x_metrics: *indexSettings
so-metrics-endpoint_x_policy: *indexSettings
so-metrics-nginx_x_stubstatus: *indexSettings
so-metrics-vsphere_x_datastore: *indexSettings
so-metrics-vsphere_x_host: *indexSettings
so-metrics-vsphere_x_virtualmachine: *indexSettings
so-case: *indexSettings
so-common: *indexSettings
so-endgame: *indexSettings
so-idh: *indexSettings
so-suricata: *indexSettings
so-suricata_x_alerts: *indexSettings
so-import: *indexSettings
so-kratos: *indexSettings
so-hydra: *indexSettings
so-kismet: *indexSettings
so-logstash: *indexSettings
so-redis: *indexSettings
so-strelka: *indexSettings
so-syslog: *indexSettings
so-zeek: *indexSettings
so-logs-system_x_auth: *dataStreamSettings
so-logs-system_x_syslog: *dataStreamSettings
so-logs-system_x_system: *dataStreamSettings
so-logs-system_x_application: *dataStreamSettings
so-logs-system_x_security: *dataStreamSettings
so-logs-windows_x_forwarded: *dataStreamSettings
so-logs-windows_x_powershell: *dataStreamSettings
so-logs-windows_x_powershell_operational: *dataStreamSettings
so-logs-windows_x_sysmon_operational: *dataStreamSettings
so-logs-winlog_x_winlog: *dataStreamSettings
so-logs-detections_x_alerts: *dataStreamSettings
so-logs-http_endpoint_x_generic: *dataStreamSettings
so-logs-httpjson_x_generic: *dataStreamSettings
so-logs-osquery-manager-actions: *dataStreamSettings
so-logs-osquery-manager-action_x_responses: *dataStreamSettings
so-logs-osquery-manager_x_action_x_responses: *dataStreamSettings
so-logs-osquery-manager_x_result: *dataStreamSettings
so-logs-elastic_agent_x_apm_server: *dataStreamSettings
so-logs-elastic_agent_x_auditbeat: *dataStreamSettings
so-logs-elastic_agent_x_cloudbeat: *dataStreamSettings
so-logs-elastic_agent_x_endpoint_security: *dataStreamSettings
so-logs-endpoint_x_alerts: *dataStreamSettings
so-logs-endpoint_x_events_x_api: *dataStreamSettings
so-logs-endpoint_x_events_x_file: *dataStreamSettings
so-logs-endpoint_x_events_x_library: *dataStreamSettings
so-logs-endpoint_x_events_x_network: *dataStreamSettings
so-logs-endpoint_x_events_x_process: *dataStreamSettings
so-logs-endpoint_x_events_x_registry: *dataStreamSettings
so-logs-endpoint_x_events_x_security: *dataStreamSettings
so-logs-elastic_agent_x_filebeat: *dataStreamSettings
so-logs-elastic_agent_x_fleet_server: *dataStreamSettings
so-logs-elastic_agent_x_heartbeat: *dataStreamSettings
so-logs-elastic_agent: *dataStreamSettings
so-logs-elastic_agent_x_metricbeat: *dataStreamSettings
so-logs-elastic_agent_x_osquerybeat: *dataStreamSettings
so-logs-elastic_agent_x_packetbeat: *dataStreamSettings
so-logs-elasticsearch_x_server: *dataStreamSettings
so-metrics-endpoint_x_metadata: *dataStreamSettings
so-metrics-endpoint_x_metrics: *dataStreamSettings
so-metrics-endpoint_x_policy: *dataStreamSettings
so-metrics-nginx_x_stubstatus: *dataStreamSettings
so-metrics-vsphere_x_datastore: *dataStreamSettings
so-metrics-vsphere_x_host: *dataStreamSettings
so-metrics-vsphere_x_virtualmachine: *dataStreamSettings
so-common: *dataStreamSettings
so-endgame: *dataStreamSettings
so-idh: *dataStreamSettings
so-suricata: *dataStreamSettings
so-suricata_x_alerts: *dataStreamSettings
so-import: *dataStreamSettings
so-kratos: *dataStreamSettings
so-hydra: *dataStreamSettings
so-kismet: *dataStreamSettings
so-logstash: *dataStreamSettings
so-redis: *dataStreamSettings
so-strelka: *dataStreamSettings
so-syslog: *dataStreamSettings
so-zeek: *dataStreamSettings
# Managed SOC integration annotations are inserted below this line. Referencing '*dataStreamSettings'
so-case: &indexSettings
index_sorting:
description: Sorts the index by event time, at the cost of additional processing resource consumption.
forcedType: bool
global: True
advanced: True
helpLink: elasticsearch
index_template:
index_patterns:
description: Patterns for matching multiple indices or tables.
forcedType: "[]string"
multiline: True
global: True
advanced: True
helpLink: elasticsearch
template:
settings:
index:
number_of_replicas:
description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs.
forcedType: int
global: True
advanced: True
helpLink: elasticsearch
auto_expand_replicas:
description: Automatically expand the number of replicas based on the number of data nodes in the cluster. This can help ensure high availability as the cluster scales up or down.
forcedType: string
regex: "^(0-[1-9]|1-[2-9]|2-[3-9]|3-[4-9]|4-[5-9]|5-[6-9]|6-[7-9]|7-[89]|8-9|[0-9]-all|false)$"
regexFailureMessage: Must be in the format of "x-y" where x is minimum number of replicas and y is maximum number of replicas, or "0-all" to specify a minimum of 0 and no maximum, or "false" to disable automatic replica expansion.
global: True
advanced: True
helpLink: elasticsearch
mapping:
total_fields:
limit:
description: Max number of fields that can exist on a single index. Larger values will consume more resources.
global: True
advanced: True
helpLink: elasticsearch
refresh_interval:
description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation.
global: True
advanced: True
helpLink: elasticsearch
number_of_shards:
description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs.
global: True
advanced: True
helpLink: elasticsearch
sort:
field:
description: The field to sort by. Must set index_sorting to True.
global: True
advanced: True
helpLink: elasticsearch
order:
description: The order to sort by. Must set index_sorting to True.
global: True
advanced: True
helpLink: elasticsearch
mappings:
_meta:
package:
name:
description: Meta settings for the mapping.
global: True
advanced: True
helpLink: elasticsearch
managed_by:
description: Meta settings for the mapping.
global: True
advanced: True
helpLink: elasticsearch
managed:
description: Meta settings for the mapping.
forcedType: bool
global: True
advanced: True
helpLink: elasticsearch
composed_of:
description: The index template is composed of these component templates.
forcedType: "[]string"
global: True
advanced: True
helpLink: elasticsearch
priority:
description: The priority of the index template.
forcedType: int
global: True
advanced: True
helpLink: elasticsearch
policy:
phases:
hot:
min_age:
description: Minimum age of index. This determines when the index should be moved to the hot tier.
global: True
advanced: True
helpLink: elasticsearch
actions:
set_priority:
priority:
description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
forcedType: int
global: True
advanced: True
helpLink: elasticsearch
rollover:
max_age:
description: Maximum age of index. Once an index reaches this limit, it will be rolled over into a new index.
global: True
advanced: True
helpLink: elasticsearch
max_primary_shard_size:
description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
global: True
advanced: True
helpLink: elasticsearch
shrink:
method:
description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
options:
- COUNT
- SIZE
global: True
advanced: True
forcedType: string
number_of_shards:
title: shard count
description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
global: True
forcedType: int
advanced: True
max_primary_shard_size:
title: max shard size
description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
regex: ^[0-9]+(?:gb|tb|pb)$
global: True
forcedType: string
advanced: True
allow_write_after_shrink:
description: Allow writes after shrink.
global: True
forcedType: bool
default: False
advanced: True
forcemerge:
max_num_segments:
description: Reduce the number of segments in each index shard and clean up deleted documents.
global: True
forcedType: int
advanced: True
index_codec:
title: compression
description: Use higher compression for stored fields at the cost of slower performance.
forcedType: bool
global: True
default: False
advanced: True
warm:
min_age:
description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier.
regex: ^[0-9]{1,5}d$
forcedType: string
global: True
advanced: True
helpLink: elasticsearch
actions:
set_priority:
priority:
description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
forcedType: int
global: True
advanced: True
helpLink: elasticsearch
rollover:
max_age:
description: Maximum age of index. Once an index reaches this limit, it will be rolled over into a new index.
global: True
advanced: True
helpLink: elasticsearch
max_primary_shard_size:
description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
global: True
advanced: True
helpLink: elasticsearch
shrink:
method:
description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
options:
- COUNT
- SIZE
global: True
advanced: True
number_of_shards:
title: shard count
description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
global: True
forcedType: int
advanced: True
max_primary_shard_size:
title: max shard size
description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
regex: ^[0-9]+(?:gb|tb|pb)$
global: True
forcedType: string
advanced: True
allow_write_after_shrink:
description: Allow writes after shrink.
global: True
forcedType: bool
default: False
advanced: True
forcemerge:
max_num_segments:
description: Reduce the number of segments in each index shard and clean up deleted documents.
global: True
forcedType: int
advanced: True
index_codec:
title: compression
description: Use higher compression for stored fields at the cost of slower performance.
forcedType: bool
global: True
default: False
advanced: True
allocate:
number_of_replicas:
description: Set the number of replicas. Remains the same as the previous phase by default.
forcedType: int
global: True
advanced: True
cold:
min_age:
description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier.
regex: ^[0-9]{1,5}d$
forcedType: string
global: True
advanced: True
helpLink: elasticsearch
actions:
set_priority:
priority:
description: Used for index recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
forcedType: int
global: True
advanced: True
helpLink: elasticsearch
allocate:
number_of_replicas:
description: Set the number of replicas. Remains the same as the previous phase by default.
forcedType: int
global: True
advanced: True
delete:
min_age:
description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion.
regex: ^[0-9]{1,5}d$
forcedType: string
global: True
advanced: True
helpLink: elasticsearch
_meta:
package:
name:
description: Meta settings for the mapping.
global: True
advanced: True
helpLink: elasticsearch
managed_by:
description: Meta settings for the mapping.
global: True
advanced: True
helpLink: elasticsearch
managed:
description: Meta settings for the mapping.
forcedType: bool
global: True
advanced: True
helpLink: elasticsearch
sos-backup: *indexSettings
so-detection: *indexSettings
so-assistant-chat: *indexSettings
so-assistant-session: *indexSettings
so-metrics-fleet_server_x_agent_status: &fleetMetricsSettings
index_sorting:
description: Sorts the index by event time, at the cost of additional processing resource consumption.
salt/elasticsearch/template.map.jinja
+15
View File
@@ -4,7 +4,11 @@
Elastic License 2.0. #}
{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
{# ELASTICSEARCHMERGED only used here to collect data_retention_method. This file intentionally works with ELASTICSEARCHDEFAULTS #}
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
{% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %}
{% set DATA_RETENTION_METHOD = ELASTICSEARCHMERGED.data_retention_method %}
{% set PILLAR_GLOBAL_OVERRIDES = {} %}
{% set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings', {}) %}
@@ -105,6 +109,17 @@
{% if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %}
{% do settings.index_template.template.settings.index.pop('sort') %}
{% endif %}
{% if DATA_RETENTION_METHOD == 'DLM' and settings.index_template.data_stream is defined and settings.data_stream_lifecycle is defined %}
{% if settings.data_stream_lifecycle.data_retention is defined and settings.data_stream_lifecycle.data_retention %}
{% do settings.index_template.template.update({'lifecycle': {'data_retention': settings.data_stream_lifecycle.data_retention}}) %}
{% else %}
{% do settings.index_template.template.update({'lifecycle': {}}) %}
{% endif %}
{% if settings.index_template.template.settings.index.lifecycle is not defined %}
{% do settings.index_template.template.settings.index.update({'lifecycle': {}}) %}
{% endif %}
{% do settings.index_template.template.settings.index.lifecycle.update({'prefer_ilm': false}) %}
{% endif %}
{% endif %}
{# advanced ilm actions #}
salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load
-8
View File
@@ -125,14 +125,6 @@ load_component_templates() {
done
}
check_elasticsearch_responsive() {
# Cannot load templates if Elasticsearch is not responding.
# NOTE: Slightly faster exit w/ failure than previous "retry 240 1" if there is a problem with Elasticsearch the
# script should exit sooner rather than hang at the 'so-elasticsearch-templates' salt state.
retry 3 15 "so-elasticsearch-query / --output /dev/null --fail" ||
fail "Elasticsearch is not responding. Please review Elasticsearch logs /opt/so/log/elasticsearch/securityonion.log for more details. Additionally, consider running so-elasticsearch-troubleshoot."
}
index_templates_exist() {
local templates_dir="$1"
salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-dlm-apply
+175
View File
@@ -0,0 +1,175 @@
#!/bin/bash
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
. /usr/sbin/so-common
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
{%- set DATA_RETENTION_METHOD = ELASTICSEARCHMERGED.data_retention_method %}
ELASTICSEARCH_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR:-/opt/so/conf/elasticsearch/templates}"
TEMPLATE_DIRS=(
"${ELASTICSEARCH_TEMPLATES_DIR}/index"
"${ELASTICSEARCH_TEMPLATES_DIR}/addon-index"
)
DATA_RETENTION_METHOD=$(cat <<'EOF'
{{ DATA_RETENTION_METHOD }}
EOF
)
DLM_FAILURES=0
DLM_FAILURE_NAMES=()
if [[ "$DATA_RETENTION_METHOD" != "DLM" && "$DATA_RETENTION_METHOD" != "ILM" ]]; then
echo "Unsupported data retention method $DATA_RETENTION_METHOD. Expected DLM or ILM."
exit 1
fi
validate_template_file() {
local template_file="$1"
if ! jq -e 'type == "object" and (.data_stream == null or (.data_stream | type == "object")) and (.template.lifecycle == null or (.template.lifecycle | type == "object")) and (.template.lifecycle.data_retention == null or (.template.lifecycle.data_retention | type == "string"))' >/dev/null 2>&1 "$template_file"; then
echo "Invalid index template JSON: $template_file"
return 1
fi
}
is_data_stream_template() {
jq -e '.data_stream | type == "object"' >/dev/null 2>&1 "$1"
}
has_data_stream_lifecycle() {
jq -e '.template.lifecycle | type == "object"' >/dev/null 2>&1 "$1"
}
get_data_retention() {
jq -r '.template.lifecycle.data_retention // ""' "$1"
}
find_template_file() {
local template="$1"
local template_dir
local template_file
for template_dir in "${TEMPLATE_DIRS[@]}"; do
template_file="${template_dir}/${template}-template.json"
if [[ -f "$template_file" ]]; then
echo "$template_file"
return 0
fi
done
return 1
}
set_data_stream_lifecycle() {
local data_stream="$1"
local data_retention="$2"
local body
local output
if [[ -n "$data_retention" ]]; then
if jq -e --arg data_stream "$data_stream" --arg data_retention "$data_retention" '.data_streams[]? | select(.name == $data_stream and .lifecycle.enabled == true and .lifecycle.data_retention == $data_retention)' >/dev/null 2>&1 <<< "$data_streams"; then
echo "DLM lifecycle already set for $data_stream with data_retention $data_retention, skipping."
return 0
fi
elif jq -e --arg data_stream "$data_stream" '.data_streams[]? | select(.name == $data_stream and .lifecycle.enabled == true and (.lifecycle.data_retention == null))' >/dev/null 2>&1 <<< "$data_streams"; then
echo "DLM lifecycle already set for $data_stream with indefinite retention, skipping."
return 0
fi
if [[ -n "$data_retention" ]]; then
body=$(jq -cn --arg data_retention "$data_retention" '{data_retention: $data_retention}')
else
# Setting indefinite retention
body='{}'
fi
if ! output=$(so-elasticsearch-query "_data_stream/${data_stream}/_lifecycle" -XPUT -d "$body" --retry 3 --retry-delay 5 --fail); then
echo "Failed to set data stream lifecycle for $data_stream."
return 1
fi
if [[ -n "$data_retention" ]]; then
echo "Set DLM lifecycle for $data_stream with data_retention $data_retention."
else
echo "Set DLM lifecycle for $data_stream with indefinite retention."
fi
}
disable_data_stream_lifecycle() {
local data_stream="$1"
local body='{"enabled":false}'
local output
if ! jq -e --arg data_stream "$data_stream" '.data_streams[]? | select(.name == $data_stream and .lifecycle != null and .lifecycle.enabled != false)' >/dev/null 2>&1 <<< "$data_streams"; then
# No action needed
return 0
fi
if ! output=$(so-elasticsearch-query "_data_stream/${data_stream}/_lifecycle" -XPUT -d "$body" --retry 3 --retry-delay 5 --fail); then
echo "Failed to disable data stream lifecycle for $data_stream."
return 1
fi
echo "Disabled DLM lifecycle for $data_stream."
}
process_data_stream() {
local data_stream="$1"
local data_retention="$2"
if [[ "$DATA_RETENTION_METHOD" == "DLM" ]]; then
set_data_stream_lifecycle "$data_stream" "$data_retention"
else
disable_data_stream_lifecycle "$data_stream"
fi
}
check_elasticsearch_responsive
if ! data_streams=$(so-elasticsearch-query "_data_stream?format=json" --retry 3 --retry-delay 5 --fail); then
echo "Failed to retrieve data streams."
exit 1
fi
while read -r data_stream_config; do
data_stream=$(jq -r '.name' <<< "$data_stream_config")
template=$(jq -r '.template' <<< "$data_stream_config")
if ! template_file=$(find_template_file "$template"); then
echo "Skipping $data_stream: index template file not found for $template."
continue
fi
validate_template_file "$template_file" || exit 1
if ! is_data_stream_template "$template_file"; then
echo "Skipping $data_stream: $template_file is not a data stream template."
continue
fi
if [[ "$DATA_RETENTION_METHOD" == "DLM" ]] && ! has_data_stream_lifecycle "$template_file"; then
echo "Skipping $data_stream: $template_file does not define data stream lifecycle."
continue
fi
data_retention=$(get_data_retention "$template_file")
if ! process_data_stream "$data_stream" "$data_retention"; then
DLM_FAILURES=$((DLM_FAILURES + 1))
DLM_FAILURE_NAMES+=("$data_stream")
fi
done < <(jq -c '.data_streams[]' <<< "$data_streams")
if [[ $DLM_FAILURES -eq 0 ]]; then
echo "Data stream lifecycle updates completed successfully."
else
echo "Encountered $DLM_FAILURES failure(s) updating data stream lifecycle:"
for failed_data_stream in "${DLM_FAILURE_NAMES[@]}"; do
echo " - $failed_data_stream"
done
exit 1
fi