CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2026-07-23 01:05:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #16099 from Security-Onion-Solutions/reyesj2-patch-7

Browse Source 
template updates
This commit is contained in:
Jorge Reyes
2026-07-22 17:40:25 -05:00
committed by GitHub
parent 4871098278 e4c14a9294
commit 19957d9530
5 changed files with 783 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade
+1 -1
View File
@@ -17,7 +17,7 @@ if INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && L
if [ "$INSTALLED_VERSION" == "$LATEST_VERSION" ]; then
echo "{{ PACKAGE }} integration version $INSTALLED_VERSION is already at the reported latest version $LATEST_VERSION, skipping upgrade."
else
echo "Upgrading {{ PACKAGE }} package to version $LATEST_VERSION..."
echo "Upgrading {{ PACKAGE }} package from $INSTALLED_VERSION to version $LATEST_VERSION..."
if ! elastic_fleet_package_install "{{ PACKAGE }}" "$LATEST_VERSION"; then
PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
salt/elasticsearch/defaults.yaml
+714
View File
@@ -3453,6 +3453,720 @@ elasticsearch:
set_priority:
priority: 50
min_age: 30d
so-metrics-system_x_core:
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- metrics@tsdb-settings
- metrics-system.core@package
- metrics@custom
- system@custom
- metrics-system.core@custom
- ecs@mappings
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates:
- metrics@custom
- system@custom
- metrics-system.core@custom
index_patterns:
- metrics-system.core-*
priority: 501
template:
settings:
index:
lifecycle:
name: so-metrics-system.core-logs
mode: time_series
number_of_replicas: 0
policy:
phases:
cold:
actions:
set_priority:
priority: 0
min_age: 60d
delete:
actions:
delete: {}
min_age: 365d
hot:
actions:
rollover:
max_age: 30d
max_primary_shard_size: 50gb
set_priority:
priority: 100
min_age: 0ms
warm:
actions:
set_priority:
priority: 50
min_age: 30d
so-metrics-system_x_cpu:
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- metrics@tsdb-settings
- metrics-system.cpu@package
- metrics@custom
- system@custom
- metrics-system.cpu@custom
- ecs@mappings
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates:
- metrics@custom
- system@custom
- metrics-system.cpu@custom
index_patterns:
- metrics-system.cpu-*
priority: 501
template:
settings:
index:
lifecycle:
name: so-metrics-system.cpu-logs
mode: time_series
number_of_replicas: 0
policy:
phases:
cold:
actions:
set_priority:
priority: 0
min_age: 60d
delete:
actions:
delete: {}
min_age: 365d
hot:
actions:
rollover:
max_age: 30d
max_primary_shard_size: 50gb
set_priority:
priority: 100
min_age: 0ms
warm:
actions:
set_priority:
priority: 50
min_age: 30d
so-metrics-system_x_diskio:
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- metrics@tsdb-settings
- metrics-system.diskio@package
- metrics@custom
- system@custom
- metrics-system.diskio@custom
- ecs@mappings
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates:
- metrics@custom
- system@custom
- metrics-system.diskio@custom
index_patterns:
- metrics-system.diskio-*
priority: 501
template:
settings:
index:
lifecycle:
name: so-metrics-system.diskio-logs
mode: time_series
number_of_replicas: 0
policy:
phases:
cold:
actions:
set_priority:
priority: 0
min_age: 60d
delete:
actions:
delete: {}
min_age: 365d
hot:
actions:
rollover:
max_age: 30d
max_primary_shard_size: 50gb
set_priority:
priority: 100
min_age: 0ms
warm:
actions:
set_priority:
priority: 50
min_age: 30d
so-metrics-system_x_filesystem:
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- metrics@tsdb-settings
- metrics-system.filesystem@package
- metrics@custom
- system@custom
- metrics-system.filesystem@custom
- ecs@mappings
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates:
- metrics@custom
- system@custom
- metrics-system.filesystem@custom
index_patterns:
- metrics-system.filesystem-*
priority: 501
template:
settings:
index:
lifecycle:
name: so-metrics-system.filesystem-logs
mode: time_series
number_of_replicas: 0
policy:
phases:
cold:
actions:
set_priority:
priority: 0
min_age: 60d
delete:
actions:
delete: {}
min_age: 365d
hot:
actions:
rollover:
max_age: 30d
max_primary_shard_size: 50gb
set_priority:
priority: 100
min_age: 0ms
warm:
actions:
set_priority:
priority: 50
min_age: 30d
so-metrics-system_x_fsstat:
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- metrics@tsdb-settings
- metrics-system.fsstat@package
- metrics@custom
- system@custom
- metrics-system.fsstat@custom
- ecs@mappings
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates:
- metrics@custom
- system@custom
- metrics-system.fsstat@custom
index_patterns:
- metrics-system.fsstat-*
priority: 501
template:
settings:
index:
lifecycle:
name: so-metrics-system.fsstat-logs
mode: time_series
number_of_replicas: 0
policy:
phases:
cold:
actions:
set_priority:
priority: 0
min_age: 60d
delete:
actions:
delete: {}
min_age: 365d
hot:
actions:
rollover:
max_age: 30d
max_primary_shard_size: 50gb
set_priority:
priority: 100
min_age: 0ms
warm:
actions:
set_priority:
priority: 50
min_age: 30d
so-metrics-system_x_load:
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- metrics@tsdb-settings
- metrics-system.load@package
- metrics@custom
- system@custom
- metrics-system.load@custom
- ecs@mappings
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates:
- metrics@custom
- system@custom
- metrics-system.load@custom
index_patterns:
- metrics-system.load-*
priority: 501
template:
settings:
index:
lifecycle:
name: so-metrics-system.load-logs
mode: time_series
number_of_replicas: 0
policy:
phases:
cold:
actions:
set_priority:
priority: 0
min_age: 60d
delete:
actions:
delete: {}
min_age: 365d
hot:
actions:
rollover:
max_age: 30d
max_primary_shard_size: 50gb
set_priority:
priority: 100
min_age: 0ms
warm:
actions:
set_priority:
priority: 50
min_age: 30d
so-metrics-system_x_memory:
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- metrics@tsdb-settings
- metrics-system.memory@package
- metrics@custom
- system@custom
- metrics-system.memory@custom
- ecs@mappings
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates:
- metrics@custom
- system@custom
- metrics-system.memory@custom
index_patterns:
- metrics-system.memory-*
priority: 501
template:
settings:
index:
lifecycle:
name: so-metrics-system.memory-logs
mode: time_series
number_of_replicas: 0
policy:
phases:
cold:
actions:
set_priority:
priority: 0
min_age: 60d
delete:
actions:
delete: {}
min_age: 365d
hot:
actions:
rollover:
max_age: 30d
max_primary_shard_size: 50gb
set_priority:
priority: 100
min_age: 0ms
warm:
actions:
set_priority:
priority: 50
min_age: 30d
so-metrics-system_x_network:
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- metrics@tsdb-settings
- metrics-system.network@package
- metrics@custom
- system@custom
- metrics-system.network@custom
- ecs@mappings
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates:
- metrics@custom
- system@custom
- metrics-system.network@custom
index_patterns:
- metrics-system.network-*
priority: 501
template:
settings:
index:
lifecycle:
name: so-metrics-system.network-logs
mode: time_series
number_of_replicas: 0
policy:
phases:
cold:
actions:
set_priority:
priority: 0
min_age: 60d
delete:
actions:
delete: {}
min_age: 365d
hot:
actions:
rollover:
max_age: 30d
max_primary_shard_size: 50gb
set_priority:
priority: 100
min_age: 0ms
warm:
actions:
set_priority:
priority: 50
min_age: 30d
so-metrics-system_x_ntp:
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- metrics@settings
- metrics-system.ntp@package
- metrics@custom
- system@custom
- metrics-system.ntp@custom
- ecs@mappings
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates:
- metrics@custom
- system@custom
- metrics-system.ntp@custom
index_patterns:
- metrics-system.ntp-*
priority: 501
template:
settings:
index:
lifecycle:
name: so-metrics-system.ntp-logs
number_of_replicas: 0
policy:
phases:
cold:
actions:
set_priority:
priority: 0
min_age: 60d
delete:
actions:
delete: {}
min_age: 365d
hot:
actions:
rollover:
max_age: 30d
max_primary_shard_size: 50gb
set_priority:
priority: 100
min_age: 0ms
warm:
actions:
set_priority:
priority: 50
min_age: 30d
so-metrics-system_x_process:
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- metrics@tsdb-settings
- metrics-system.process@package
- metrics@custom
- system@custom
- metrics-system.process@custom
- ecs@mappings
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates:
- metrics@custom
- system@custom
- metrics-system.process@custom
index_patterns:
- metrics-system.process-*
priority: 501
template:
settings:
index:
lifecycle:
name: so-metrics-system.process-logs
mode: time_series
number_of_replicas: 0
policy:
phases:
cold:
actions:
set_priority:
priority: 0
min_age: 60d
delete:
actions:
delete: {}
min_age: 365d
hot:
actions:
rollover:
max_age: 30d
max_primary_shard_size: 50gb
set_priority:
priority: 100
min_age: 0ms
warm:
actions:
set_priority:
priority: 50
min_age: 30d
so-metrics-system_x_process_x_summary:
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- metrics@tsdb-settings
- metrics-system.process.summary@package
- metrics@custom
- system@custom
- metrics-system.process.summary@custom
- ecs@mappings
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates:
- metrics@custom
- system@custom
- metrics-system.process.summary@custom
index_patterns:
- metrics-system.process.summary-*
priority: 501
template:
settings:
index:
lifecycle:
name: so-metrics-system.process.summary-logs
mode: time_series
number_of_replicas: 0
policy:
phases:
cold:
actions:
set_priority:
priority: 0
min_age: 60d
delete:
actions:
delete: {}
min_age: 365d
hot:
actions:
rollover:
max_age: 30d
max_primary_shard_size: 50gb
set_priority:
priority: 100
min_age: 0ms
warm:
actions:
set_priority:
priority: 50
min_age: 30d
so-metrics-system_x_socket_summary:
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- metrics@tsdb-settings
- metrics-system.socket_summary@package
- metrics@custom
- system@custom
- metrics-system.socket_summary@custom
- ecs@mappings
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates:
- metrics@custom
- system@custom
- metrics-system.socket_summary@custom
index_patterns:
- metrics-system.socket_summary-*
priority: 501
template:
settings:
index:
lifecycle:
name: so-metrics-system.socket_summary-logs
mode: time_series
number_of_replicas: 0
policy:
phases:
cold:
actions:
set_priority:
priority: 0
min_age: 60d
delete:
actions:
delete: {}
min_age: 365d
hot:
actions:
rollover:
max_age: 30d
max_primary_shard_size: 50gb
set_priority:
priority: 100
min_age: 0ms
warm:
actions:
set_priority:
priority: 50
min_age: 30d
so-metrics-system_x_uptime:
data_stream_lifecycle:
data_retention: 90d
index_template:
composed_of:
- metrics@tsdb-settings
- metrics-system.uptime@package
- metrics@custom
- system@custom
- metrics-system.uptime@custom
- ecs@mappings
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
- so-fleet_agent_id_verification-1
data_stream:
allow_custom_routing: false
hidden: false
ignore_missing_component_templates:
- metrics@custom
- system@custom
- metrics-system.uptime@custom
index_patterns:
- metrics-system.uptime-*
priority: 501
template:
settings:
index:
lifecycle:
name: so-metrics-system.uptime-logs
mode: time_series
number_of_replicas: 0
policy:
phases:
cold:
actions:
set_priority:
priority: 0
min_age: 60d
delete:
actions:
delete: {}
min_age: 365d
hot:
actions:
rollover:
max_age: 30d
max_primary_shard_size: 50gb
set_priority:
priority: 100
min_age: 0ms
warm:
actions:
set_priority:
priority: 50
min_age: 30d
so-logs-windows_x_forwarded:
index_sorting: false
data_stream_lifecycle:
salt/elasticsearch/files/log4j2.properties
+2 -1
View File
@@ -20,7 +20,8 @@ appender.rolling.strategy.type = DefaultRolloverStrategy
appender.rolling.strategy.action.type = Delete
appender.rolling.strategy.action.basepath = /var/log/elasticsearch
appender.rolling.strategy.action.condition.type = IfFileName
appender.rolling.strategy.action.condition.glob = *.log.gz
# age delete regular securityonion.log.gz and gc.log.NN files
appender.rolling.strategy.action.condition.regex = (?:.*[.]log[.]gz|gc[.]log[.][0-9]+)
appender.rolling.strategy.action.condition.nested_condition.type = IfLastModified
appender.rolling.strategy.action.condition.nested_condition.age = 7D
salt/elasticsearch/soc_elasticsearch.yaml
+57 -13
View File
@@ -64,6 +64,43 @@ elasticsearch:
flood_stage:
description: The max percentage of used disk space that will cause the node to take protective actions, such as blocking incoming events.
helpLink: elasticsearch
lifecycle:
default:
rollover:
description: This property accepts a key value pair formatted string and configures the conditions that would trigger a data stream to rollover when it has lifecycle configured.
forcedType: string
regex: ^max_age=(auto|[1-9][0-9]*[hd]),max_primary_shard_size=[1-9][0-9]*gb,min_docs=(0|[1-9][0-9]*),max_primary_shard_docs=[1-9][0-9]*$
regexFailureMessage: Must be in the format of "max_age=auto|<number><h|d>,max_primary_shard_size=<number>gb,min_docs=<number>,max_primary_shard_docs=<number>".
advanced: True
global: True
data_streams:
lifecycle:
poll_interval:
description: How often Elasticsearch checks what the next action is for all data streams with a built-in lifecycle.
forcedType: string
regex: "^[1-9][0-9]*[mhd]$"
regexFailureMessage: Must be a number followed by m, h, or d.
advanced: True
global: true
helpLink: elasticsearch
target:
merge:
policy:
merge_factor:
description: Data stream lifecycle implements tail merging by updating the Lucene merge policy factor for the target backing index. The merge factor is both the number of segments that should be merged together, and the maximum number of segments that we expect to find.
forcedType: int
regex: "^[1-9][0-9]*$"
advanced: True
global: true
helpLink: elasticsearch
floor_segment:
description: Data stream lifecycle implements tail merging by updating the Lucene merge policy floor segment for the target backing index. This floor segment size is a way to prevent indices from having a long tail of very small segments.
forcedType: string
regex: "^[1-9][0-9]*[MG]B$"
regexFailureMessage: Must be a number followed by MB or GB, such as 100MB.
advanced: True
global: true
helpLink: elasticsearch
action:
destructive_requires_name:
description: Requires explicit index names when deleting indices. Prevents accidental deletion of indices via wildcard patterns.
@@ -645,6 +682,9 @@ elasticsearch:
global: True
advanced: True
helpLink: elasticsearch
so-assistant-chat: *dataStreamSettings
so-assistant-session: *dataStreamSettings
so-elastic-agent-monitor: *dataStreamSettings
so-logs-soc: *dataStreamSettings
so-logs-system_x_auth: *dataStreamSettings
so-logs-system_x_syslog: *dataStreamSettings
@@ -667,7 +707,10 @@ elasticsearch:
so-logs-elastic_agent_x_auditbeat: *dataStreamSettings
so-logs-elastic_agent_x_cloudbeat: *dataStreamSettings
so-logs-elastic_agent_x_endpoint_security: *dataStreamSettings
so-logs-endpoint_x_actions: *dataStreamSettings
so-logs-endpoint_x_action_x_responses: *dataStreamSettings
so-logs-endpoint_x_alerts: *dataStreamSettings
so-logs-endpoint_x_diagnostic_x_collection: *dataStreamSettings
so-logs-endpoint_x_events_x_api: *dataStreamSettings
so-logs-endpoint_x_events_x_file: *dataStreamSettings
so-logs-endpoint_x_events_x_library: *dataStreamSettings
@@ -675,6 +718,7 @@ elasticsearch:
so-logs-endpoint_x_events_x_process: *dataStreamSettings
so-logs-endpoint_x_events_x_registry: *dataStreamSettings
so-logs-endpoint_x_events_x_security: *dataStreamSettings
so-logs-endpoint_x_heartbeat: *dataStreamSettings
so-logs-elastic_agent_x_filebeat: *dataStreamSettings
so-logs-elastic_agent_x_fleet_server: *dataStreamSettings
so-logs-elastic_agent_x_heartbeat: *dataStreamSettings
@@ -690,6 +734,19 @@ elasticsearch:
so-metrics-vsphere_x_datastore: *dataStreamSettings
so-metrics-vsphere_x_host: *dataStreamSettings
so-metrics-vsphere_x_virtualmachine: *dataStreamSettings
so-metrics-system_x_core: *dataStreamSettings
so-metrics-system_x_cpu: *dataStreamSettings
so-metrics-system_x_diskio: *dataStreamSettings
so-metrics-system_x_filesystem: *dataStreamSettings
so-metrics-system_x_fsstat: *dataStreamSettings
so-metrics-system_x_load: *dataStreamSettings
so-metrics-system_x_memory: *dataStreamSettings
so-metrics-system_x_network: *dataStreamSettings
so-metrics-system_x_ntp: *dataStreamSettings
so-metrics-system_x_process: *dataStreamSettings
so-metrics-system_x_process_x_summary: *dataStreamSettings
so-metrics-system_x_socket_summary: *dataStreamSettings
so-metrics-system_x_uptime: *dataStreamSettings
so-common: *dataStreamSettings
so-endgame: *dataStreamSettings
so-idh: *dataStreamSettings
@@ -880,17 +937,6 @@ elasticsearch:
global: True
advanced: True
helpLink: elasticsearch
rollover:
max_age:
description: Maximum age of index. Once an index reaches this limit, it will be rolled over into a new index.
global: True
advanced: True
helpLink: elasticsearch
max_primary_shard_size:
description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
global: True
advanced: True
helpLink: elasticsearch
shrink:
method:
description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
@@ -987,8 +1033,6 @@ elasticsearch:
helpLink: elasticsearch
sos-backup: *indexSettings
so-detection: *indexSettings
so-assistant-chat: *indexSettings
so-assistant-session: *indexSettings
so-metrics-fleet_server_x_agent_status: &fleetMetricsSettings
index_sorting:
description: Sorts the index by event time, at the cost of additional processing resource consumption.
salt/elasticsearch/template.map.jinja
+9 -3
View File
@@ -109,9 +109,15 @@
{% if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %}
{% do settings.index_template.template.settings.index.pop('sort') %}
{% endif %}
{% if DATA_RETENTION_METHOD == 'DLM' and settings.index_template.data_stream is defined and settings.data_stream_lifecycle is defined %}
{% if settings.data_stream_lifecycle.data_retention is defined and settings.data_stream_lifecycle.data_retention %}
{% do settings.index_template.template.update({'lifecycle': {'data_retention': settings.data_stream_lifecycle.data_retention}}) %}
{% if DATA_RETENTION_METHOD == 'DLM' and settings.index_template.data_stream is defined %}
{# Addon defaults are generated without data_stream_lifecycle, so fall back to global defaults. #}
{% if settings.data_stream_lifecycle is defined %}
{% set DATA_STREAM_LIFECYCLE = settings.data_stream_lifecycle %}
{% else %}
{% set DATA_STREAM_LIFECYCLE = DEFAULT_GLOBAL_OVERRIDES.data_stream_lifecycle %}
{% endif %}
{% if DATA_STREAM_LIFECYCLE.data_retention is defined and DATA_STREAM_LIFECYCLE.data_retention %}
{% do settings.index_template.template.update({'lifecycle': {'data_retention': DATA_STREAM_LIFECYCLE.data_retention}}) %}
{% else %}
{% do settings.index_template.template.update({'lifecycle': {}}) %}
{% endif %}