diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade index f3d77b852..d645403d5 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade @@ -17,7 +17,7 @@ if INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && L if [ "$INSTALLED_VERSION" == "$LATEST_VERSION" ]; then echo "{{ PACKAGE }} integration version $INSTALLED_VERSION is already at the reported latest version $LATEST_VERSION, skipping upgrade." else - echo "Upgrading {{ PACKAGE }} package to version $LATEST_VERSION..." + echo "Upgrading {{ PACKAGE }} package from $INSTALLED_VERSION to version $LATEST_VERSION..." if ! elastic_fleet_package_install "{{ PACKAGE }}" "$LATEST_VERSION"; then PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1)) PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}") diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 14d753c21..5179fac2b 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -3453,6 +3453,720 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-metrics-system_x_core: + data_stream_lifecycle: + data_retention: 90d + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-system.core@package + - metrics@custom + - system@custom + - metrics-system.core@custom + - ecs@mappings + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics@custom + - system@custom + - metrics-system.core@custom + index_patterns: + - metrics-system.core-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-system.core-logs + mode: time_series + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-system_x_cpu: + data_stream_lifecycle: + data_retention: 90d + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-system.cpu@package + - metrics@custom + - system@custom + - metrics-system.cpu@custom + - ecs@mappings + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics@custom + - system@custom + - metrics-system.cpu@custom + index_patterns: + - metrics-system.cpu-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-system.cpu-logs + mode: time_series + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-system_x_diskio: + data_stream_lifecycle: + data_retention: 90d + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-system.diskio@package + - metrics@custom + - system@custom + - metrics-system.diskio@custom + - ecs@mappings + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics@custom + - system@custom + - metrics-system.diskio@custom + index_patterns: + - metrics-system.diskio-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-system.diskio-logs + mode: time_series + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-system_x_filesystem: + data_stream_lifecycle: + data_retention: 90d + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-system.filesystem@package + - metrics@custom + - system@custom + - metrics-system.filesystem@custom + - ecs@mappings + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics@custom + - system@custom + - metrics-system.filesystem@custom + index_patterns: + - metrics-system.filesystem-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-system.filesystem-logs + mode: time_series + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-system_x_fsstat: + data_stream_lifecycle: + data_retention: 90d + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-system.fsstat@package + - metrics@custom + - system@custom + - metrics-system.fsstat@custom + - ecs@mappings + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics@custom + - system@custom + - metrics-system.fsstat@custom + index_patterns: + - metrics-system.fsstat-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-system.fsstat-logs + mode: time_series + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-system_x_load: + data_stream_lifecycle: + data_retention: 90d + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-system.load@package + - metrics@custom + - system@custom + - metrics-system.load@custom + - ecs@mappings + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics@custom + - system@custom + - metrics-system.load@custom + index_patterns: + - metrics-system.load-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-system.load-logs + mode: time_series + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-system_x_memory: + data_stream_lifecycle: + data_retention: 90d + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-system.memory@package + - metrics@custom + - system@custom + - metrics-system.memory@custom + - ecs@mappings + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics@custom + - system@custom + - metrics-system.memory@custom + index_patterns: + - metrics-system.memory-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-system.memory-logs + mode: time_series + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-system_x_network: + data_stream_lifecycle: + data_retention: 90d + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-system.network@package + - metrics@custom + - system@custom + - metrics-system.network@custom + - ecs@mappings + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics@custom + - system@custom + - metrics-system.network@custom + index_patterns: + - metrics-system.network-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-system.network-logs + mode: time_series + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-system_x_ntp: + data_stream_lifecycle: + data_retention: 90d + index_template: + composed_of: + - metrics@settings + - metrics-system.ntp@package + - metrics@custom + - system@custom + - metrics-system.ntp@custom + - ecs@mappings + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics@custom + - system@custom + - metrics-system.ntp@custom + index_patterns: + - metrics-system.ntp-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-system.ntp-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-system_x_process: + data_stream_lifecycle: + data_retention: 90d + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-system.process@package + - metrics@custom + - system@custom + - metrics-system.process@custom + - ecs@mappings + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics@custom + - system@custom + - metrics-system.process@custom + index_patterns: + - metrics-system.process-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-system.process-logs + mode: time_series + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-system_x_process_x_summary: + data_stream_lifecycle: + data_retention: 90d + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-system.process.summary@package + - metrics@custom + - system@custom + - metrics-system.process.summary@custom + - ecs@mappings + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics@custom + - system@custom + - metrics-system.process.summary@custom + index_patterns: + - metrics-system.process.summary-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-system.process.summary-logs + mode: time_series + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-system_x_socket_summary: + data_stream_lifecycle: + data_retention: 90d + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-system.socket_summary@package + - metrics@custom + - system@custom + - metrics-system.socket_summary@custom + - ecs@mappings + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics@custom + - system@custom + - metrics-system.socket_summary@custom + index_patterns: + - metrics-system.socket_summary-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-system.socket_summary-logs + mode: time_series + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-system_x_uptime: + data_stream_lifecycle: + data_retention: 90d + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-system.uptime@package + - metrics@custom + - system@custom + - metrics-system.uptime@custom + - ecs@mappings + - so-fleet_integrations.ip_mappings-1 + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics@custom + - system@custom + - metrics-system.uptime@custom + index_patterns: + - metrics-system.uptime-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-system.uptime-logs + mode: time_series + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-windows_x_forwarded: index_sorting: false data_stream_lifecycle: diff --git a/salt/elasticsearch/files/log4j2.properties b/salt/elasticsearch/files/log4j2.properties index 050071581..a938b6247 100644 --- a/salt/elasticsearch/files/log4j2.properties +++ b/salt/elasticsearch/files/log4j2.properties @@ -20,7 +20,8 @@ appender.rolling.strategy.type = DefaultRolloverStrategy appender.rolling.strategy.action.type = Delete appender.rolling.strategy.action.basepath = /var/log/elasticsearch appender.rolling.strategy.action.condition.type = IfFileName -appender.rolling.strategy.action.condition.glob = *.log.gz +# age delete regular securityonion.log.gz and gc.log.NN files +appender.rolling.strategy.action.condition.regex = (?:.*[.]log[.]gz|gc[.]log[.][0-9]+) appender.rolling.strategy.action.condition.nested_condition.type = IfLastModified appender.rolling.strategy.action.condition.nested_condition.age = 7D diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 95880bb9d..a885f15bc 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -64,6 +64,43 @@ elasticsearch: flood_stage: description: The max percentage of used disk space that will cause the node to take protective actions, such as blocking incoming events. helpLink: elasticsearch + lifecycle: + default: + rollover: + description: This property accepts a key value pair formatted string and configures the conditions that would trigger a data stream to rollover when it has lifecycle configured. + forcedType: string + regex: ^max_age=(auto|[1-9][0-9]*[hd]),max_primary_shard_size=[1-9][0-9]*gb,min_docs=(0|[1-9][0-9]*),max_primary_shard_docs=[1-9][0-9]*$ + regexFailureMessage: Must be in the format of "max_age=auto|,max_primary_shard_size=gb,min_docs=,max_primary_shard_docs=". + advanced: True + global: True + data_streams: + lifecycle: + poll_interval: + description: How often Elasticsearch checks what the next action is for all data streams with a built-in lifecycle. + forcedType: string + regex: "^[1-9][0-9]*[mhd]$" + regexFailureMessage: Must be a number followed by m, h, or d. + advanced: True + global: true + helpLink: elasticsearch + target: + merge: + policy: + merge_factor: + description: Data stream lifecycle implements tail merging by updating the Lucene merge policy factor for the target backing index. The merge factor is both the number of segments that should be merged together, and the maximum number of segments that we expect to find. + forcedType: int + regex: "^[1-9][0-9]*$" + advanced: True + global: true + helpLink: elasticsearch + floor_segment: + description: Data stream lifecycle implements tail merging by updating the Lucene merge policy floor segment for the target backing index. This floor segment size is a way to prevent indices from having a long tail of very small segments. + forcedType: string + regex: "^[1-9][0-9]*[MG]B$" + regexFailureMessage: Must be a number followed by MB or GB, such as 100MB. + advanced: True + global: true + helpLink: elasticsearch action: destructive_requires_name: description: Requires explicit index names when deleting indices. Prevents accidental deletion of indices via wildcard patterns. @@ -645,6 +682,9 @@ elasticsearch: global: True advanced: True helpLink: elasticsearch + so-assistant-chat: *dataStreamSettings + so-assistant-session: *dataStreamSettings + so-elastic-agent-monitor: *dataStreamSettings so-logs-soc: *dataStreamSettings so-logs-system_x_auth: *dataStreamSettings so-logs-system_x_syslog: *dataStreamSettings @@ -667,7 +707,10 @@ elasticsearch: so-logs-elastic_agent_x_auditbeat: *dataStreamSettings so-logs-elastic_agent_x_cloudbeat: *dataStreamSettings so-logs-elastic_agent_x_endpoint_security: *dataStreamSettings + so-logs-endpoint_x_actions: *dataStreamSettings + so-logs-endpoint_x_action_x_responses: *dataStreamSettings so-logs-endpoint_x_alerts: *dataStreamSettings + so-logs-endpoint_x_diagnostic_x_collection: *dataStreamSettings so-logs-endpoint_x_events_x_api: *dataStreamSettings so-logs-endpoint_x_events_x_file: *dataStreamSettings so-logs-endpoint_x_events_x_library: *dataStreamSettings @@ -675,6 +718,7 @@ elasticsearch: so-logs-endpoint_x_events_x_process: *dataStreamSettings so-logs-endpoint_x_events_x_registry: *dataStreamSettings so-logs-endpoint_x_events_x_security: *dataStreamSettings + so-logs-endpoint_x_heartbeat: *dataStreamSettings so-logs-elastic_agent_x_filebeat: *dataStreamSettings so-logs-elastic_agent_x_fleet_server: *dataStreamSettings so-logs-elastic_agent_x_heartbeat: *dataStreamSettings @@ -690,6 +734,19 @@ elasticsearch: so-metrics-vsphere_x_datastore: *dataStreamSettings so-metrics-vsphere_x_host: *dataStreamSettings so-metrics-vsphere_x_virtualmachine: *dataStreamSettings + so-metrics-system_x_core: *dataStreamSettings + so-metrics-system_x_cpu: *dataStreamSettings + so-metrics-system_x_diskio: *dataStreamSettings + so-metrics-system_x_filesystem: *dataStreamSettings + so-metrics-system_x_fsstat: *dataStreamSettings + so-metrics-system_x_load: *dataStreamSettings + so-metrics-system_x_memory: *dataStreamSettings + so-metrics-system_x_network: *dataStreamSettings + so-metrics-system_x_ntp: *dataStreamSettings + so-metrics-system_x_process: *dataStreamSettings + so-metrics-system_x_process_x_summary: *dataStreamSettings + so-metrics-system_x_socket_summary: *dataStreamSettings + so-metrics-system_x_uptime: *dataStreamSettings so-common: *dataStreamSettings so-endgame: *dataStreamSettings so-idh: *dataStreamSettings @@ -880,17 +937,6 @@ elasticsearch: global: True advanced: True helpLink: elasticsearch - rollover: - max_age: - description: Maximum age of index. Once an index reaches this limit, it will be rolled over into a new index. - global: True - advanced: True - helpLink: elasticsearch - max_primary_shard_size: - description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index. - global: True - advanced: True - helpLink: elasticsearch shrink: method: description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size. @@ -987,8 +1033,6 @@ elasticsearch: helpLink: elasticsearch sos-backup: *indexSettings so-detection: *indexSettings - so-assistant-chat: *indexSettings - so-assistant-session: *indexSettings so-metrics-fleet_server_x_agent_status: &fleetMetricsSettings index_sorting: description: Sorts the index by event time, at the cost of additional processing resource consumption. diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index 72be1ec58..c460f4ab1 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -109,9 +109,15 @@ {% if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %} {% do settings.index_template.template.settings.index.pop('sort') %} {% endif %} -{% if DATA_RETENTION_METHOD == 'DLM' and settings.index_template.data_stream is defined and settings.data_stream_lifecycle is defined %} -{% if settings.data_stream_lifecycle.data_retention is defined and settings.data_stream_lifecycle.data_retention %} -{% do settings.index_template.template.update({'lifecycle': {'data_retention': settings.data_stream_lifecycle.data_retention}}) %} +{% if DATA_RETENTION_METHOD == 'DLM' and settings.index_template.data_stream is defined %} +{# Addon defaults are generated without data_stream_lifecycle, so fall back to global defaults. #} +{% if settings.data_stream_lifecycle is defined %} +{% set DATA_STREAM_LIFECYCLE = settings.data_stream_lifecycle %} +{% else %} +{% set DATA_STREAM_LIFECYCLE = DEFAULT_GLOBAL_OVERRIDES.data_stream_lifecycle %} +{% endif %} +{% if DATA_STREAM_LIFECYCLE.data_retention is defined and DATA_STREAM_LIFECYCLE.data_retention %} +{% do settings.index_template.template.update({'lifecycle': {'data_retention': DATA_STREAM_LIFECYCLE.data_retention}}) %} {% else %} {% do settings.index_template.template.update({'lifecycle': {}}) %} {% endif %}