From dff3d76efd73d869bdf0532dd9d952b6a1802045 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 19 Aug 2026 16:21:28 -0400 Subject: [PATCH 1/5] Expose Logstash 9.3.7 pipeline settings per pipeline in SOC Add logstash:pipeline_settings carrying the 27 pipeline-scoped settings Logstash 9.3.7 accepts, annotated individually per pipeline and rendered into pipelines.yml. A blank setting inherits from logstash.yml. Restart logstash when pipelines.yml changes, and add the missing managerhype annotation. Fixes #15090 --- salt/logstash/config.sls | 8 +- salt/logstash/defaults.yaml | 253 ++++++++++++++++++++ salt/logstash/enabled.sls | 1 + salt/logstash/etc/pipelines.yml.jinja | 13 ++ salt/logstash/soc_logstash.yaml | 317 ++++++++++++++++++++++++++ 5 files changed, 590 insertions(+), 2 deletions(-) diff --git a/salt/logstash/config.sls b/salt/logstash/config.sls index 47feba42c..a661e96cc 100644 --- a/salt/logstash/config.sls +++ b/salt/logstash/config.sls @@ -81,6 +81,10 @@ ls_custom_pipeline_conf_{{assigned_pipeline}}_{{pipeline}}: {% for assigned_pipeline in ASSIGNED_PIPELINES %} +{# a blank per-pipeline setting falls back to the global logstash.yml value #} +{% set PIPELINE_OVERRIDES = LOGSTASH_MERGED.get('pipeline_settings', {}).get(assigned_pipeline, {}) %} +{% set THREADS = PIPELINE_OVERRIDES.get('pipeline_x_workers') or LOGSTASH_MERGED.config.pipeline_x_workers %} +{% set BATCH = PIPELINE_OVERRIDES.get('pipeline_x_batch_x_size') or LOGSTASH_MERGED.config.pipeline_x_batch_x_size %} {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %} ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}: file.managed: @@ -92,8 +96,8 @@ ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") GLOBALS: {{ GLOBALS }} ES_USER: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') }}" ES_PASS: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') }}" - THREADS: {{ LOGSTASH_MERGED.config.pipeline_x_workers }} - BATCH: {{ LOGSTASH_MERGED.config.pipeline_x_batch_x_size }} + THREADS: {{ THREADS }} + BATCH: {{ BATCH }} {% else %} - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{CONFIGFILE.split('/')[1]}} {% endif %} diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index db5e4ee58..84bb91afd 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -60,6 +60,259 @@ logstash: custom008: PLACEHOLDER custom009: PLACEHOLDER custom010: PLACEHOLDER + pipeline_settings: + fleet: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' + manager: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' + receiver: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' + search: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' + custom0: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' + custom1: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' + custom2: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' + custom3: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' + custom4: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' settings: lsheap: 500m config: diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index 80e40d78d..fb0852f00 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -105,6 +105,7 @@ so-logstash: {% endif %} - watch: - file: lsetcsync + - file: lspipelinesyml - file: trusttheca {% if GLOBALS.is_manager %} - file: elasticsearch_cacerts diff --git a/salt/logstash/etc/pipelines.yml.jinja b/salt/logstash/etc/pipelines.yml.jinja index 427cc9f14..7788ba601 100644 --- a/salt/logstash/etc/pipelines.yml.jinja +++ b/salt/logstash/etc/pipelines.yml.jinja @@ -1,4 +1,17 @@ +{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %} +{%- set PIPELINE_SETTINGS = LOGSTASH_MERGED.get('pipeline_settings', {}) %} {%- for assigned_pipeline in ASSIGNED_PIPELINES %} - pipeline.id: {{ assigned_pipeline }} path.config: "/usr/share/logstash/pipelines/{{ assigned_pipeline }}/" +{%- set extra = PIPELINE_SETTINGS.get(assigned_pipeline, {}) %} +{%- if extra is mapping %} +{#- values are emitted unquoted so yaml re-infers the type logstash expects: + 4 as an integer, false as a boolean, 1024mb and auto as strings #} +{%- for key, value in extra | dictsort %} +{%- set rendered = key | replace('_x_', '.') %} +{%- if value not in ['', None] and rendered not in ['pipeline.id', 'path.config'] %} + {{ rendered }}: {{ value }} +{%- endif %} +{%- endfor %} +{%- endif %} {% endfor -%} diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index 40794afe4..9dfad556b 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -16,6 +16,7 @@ logstash: heavynode: *assigned_pipelines searchnode: *assigned_pipelines manager: *assigned_pipelines + managerhype: *assigned_pipelines managersearch: *assigned_pipelines fleet: *assigned_pipelines defined_pipelines: @@ -51,6 +52,322 @@ logstash: custom008: *pipeline_config custom009: *pipeline_config custom010: *pipeline_config + pipeline_settings: + manager: &pipeline_settings + pipeline_x_workers: + description: >- + Number of worker threads that run filters and outputs for this pipeline. May be set higher + than the CPU core count when outputs spend time waiting on I/O. Leave blank to use the value + from logstash.yml. + title: pipeline.workers + regex: '^$|^[1-9][0-9]*$' + regexFailureMessage: Must be blank, or a positive whole number. + advanced: True + global: False + helpLink: logstash + pipeline_x_batch_x_size: + description: >- + Maximum number of events an individual worker thread collects before running filters and + outputs. Larger batches are more efficient but increase heap use; total in-flight events is + workers multiplied by batch size. Leave blank to use the value from logstash.yml. + title: pipeline.batch.size + regex: '^$|^[1-9][0-9]*$' + regexFailureMessage: Must be blank, or a positive whole number. + advanced: True + global: False + helpLink: logstash + pipeline_x_batch_x_delay: + description: >- + Milliseconds a worker waits for the next event before running a batch that is not yet full. + Leave blank to use the value from logstash.yml. + title: pipeline.batch.delay + regex: '^$|^[0-9]+$' + regexFailureMessage: Must be blank, or a whole number. + advanced: True + global: False + helpLink: logstash + pipeline_x_batch_x_metrics_x_sampling_mode: + description: >- + How much batch size metering this pipeline records. Fuller sampling helps size batches but + consumes additional heap. Leave blank to use the value from logstash.yml. + title: pipeline.batch.metrics.sampling_mode + options: + - '' + - 'disabled' + - 'minimal' + - 'full' + advanced: True + global: False + helpLink: logstash + pipeline_x_ordered: + description: >- + Whether event order is preserved through this pipeline. auto enables ordering only when the + pipeline runs a single worker. Leave blank to use the value from logstash.yml. + title: pipeline.ordered + options: + - '' + - 'auto' + - 'true' + - 'false' + advanced: True + global: False + helpLink: logstash + pipeline_x_ecs_compatibility: + description: >- + Elastic Common Schema compatibility mode for plugins in this pipeline. Security Onion sets + this globally and it should rarely be changed per pipeline. Leave blank to use the value + from logstash.yml. + title: pipeline.ecs_compatibility + options: + - '' + - 'disabled' + - 'v1' + - 'v8' + advanced: True + global: False + helpLink: logstash + pipeline_x_reloadable: + description: >- + Whether this pipeline may be reloaded when its configuration changes. Leave blank to use the + value from logstash.yml. + title: pipeline.reloadable + options: + - '' + - 'true' + - 'false' + advanced: True + global: False + helpLink: logstash + queue_x_type: + description: >- + Queue backing this pipeline. persisted buffers events to disk under /nsm/logstash so they + survive a restart, at some throughput cost; memory does not. Leave blank to use the value + from logstash.yml. + title: queue.type + options: + - '' + - 'memory' + - 'persisted' + advanced: True + global: False + helpLink: logstash + queue_x_max_bytes: + description: >- + Total size of the persistent queue for this pipeline. Only applies when queue.type is + persisted, and must fit the disk backing /nsm/logstash. Leave blank to use the value from + logstash.yml. + title: queue.max_bytes + regex: '^$|^[0-9]+(b|kb|mb|gb|tb|pb)$' + regexFailureMessage: Must be blank, or a size such as 512mb, 1gb. + advanced: True + global: False + helpLink: logstash + queue_x_page_capacity: + description: >- + Size of each page in the persistent queue for this pipeline. Leave blank to use the value + from logstash.yml. + title: queue.page_capacity + regex: '^$|^[0-9]+(b|kb|mb|gb|tb|pb)$' + regexFailureMessage: Must be blank, or a size such as 512mb, 1gb. + advanced: True + global: False + helpLink: logstash + queue_x_max_events: + description: >- + Maximum number of events in the persistent queue for this pipeline. 0 means unlimited. Leave + blank to use the value from logstash.yml. + title: queue.max_events + regex: '^$|^[0-9]+$' + regexFailureMessage: Must be blank, or a whole number. + advanced: True + global: False + helpLink: logstash + queue_x_checkpoint_x_acks: + description: >- + Number of acknowledged events before a persistent queue checkpoint is forced. 0 means + unlimited. Leave blank to use the value from logstash.yml. + title: queue.checkpoint.acks + regex: '^$|^[0-9]+$' + regexFailureMessage: Must be blank, or a whole number. + advanced: True + global: False + helpLink: logstash + queue_x_checkpoint_x_writes: + description: >- + Number of written events before a persistent queue checkpoint is forced. 0 means unlimited. + Leave blank to use the value from logstash.yml. + title: queue.checkpoint.writes + regex: '^$|^[0-9]+$' + regexFailureMessage: Must be blank, or a whole number. + advanced: True + global: False + helpLink: logstash + queue_x_checkpoint_x_interval: + description: >- + Milliseconds between persistent queue head page checkpoints. 0 disables periodic + checkpointing. Leave blank to use the value from logstash.yml. + title: queue.checkpoint.interval + regex: '^$|^[0-9]+$' + regexFailureMessage: Must be blank, or a whole number. + advanced: True + global: False + helpLink: logstash + queue_x_checkpoint_x_retry: + description: >- + Whether Logstash retries a failed persistent queue checkpoint write. Leave blank to use the + value from logstash.yml. + title: queue.checkpoint.retry + options: + - '' + - 'true' + - 'false' + advanced: True + global: False + helpLink: logstash + queue_x_compression: + description: >- + Compression applied to persistent queue pages for this pipeline, trading CPU for disk. Leave + blank to use the value from logstash.yml. + title: queue.compression + options: + - '' + - 'none' + - 'speed' + - 'balanced' + - 'size' + - 'disabled' + advanced: True + global: False + helpLink: logstash + queue_x_drain: + description: >- + Whether Logstash drains the persistent queue before shutting down this pipeline. Draining a + large queue makes shutdown take considerably longer. Leave blank to use the value from + logstash.yml. + title: queue.drain + options: + - '' + - 'true' + - 'false' + advanced: True + global: False + helpLink: logstash + dead_letter_queue_x_enable: + description: >- + Whether events this pipeline cannot process are written to a dead letter queue instead of + being dropped. Leave blank to use the value from logstash.yml. + title: dead_letter_queue.enable + options: + - '' + - 'true' + - 'false' + advanced: True + global: False + helpLink: logstash + dead_letter_queue_x_max_bytes: + description: >- + Total size of the dead letter queue for this pipeline. Leave blank to use the value from + logstash.yml. + title: dead_letter_queue.max_bytes + regex: '^$|^[0-9]+(b|kb|mb|gb|tb|pb)$' + regexFailureMessage: Must be blank, or a size such as 512mb, 1gb. + advanced: True + global: False + helpLink: logstash + dead_letter_queue_x_flush_interval: + description: >- + Milliseconds before a partial dead letter queue segment is flushed. Leave blank to use the + value from logstash.yml. + title: dead_letter_queue.flush_interval + regex: '^$|^[0-9]+$' + regexFailureMessage: Must be blank, or a whole number. + advanced: True + global: False + helpLink: logstash + dead_letter_queue_x_flush_check_interval: + description: >- + Milliseconds between checks for a dead letter queue segment that needs flushing. Leave blank + to use the value from logstash.yml. + title: dead_letter_queue.flush_check_interval + regex: '^$|^[0-9]+$' + regexFailureMessage: Must be blank, or a whole number. + advanced: True + global: False + helpLink: logstash + dead_letter_queue_x_storage_policy: + description: >- + What happens when the dead letter queue is full: drop_newer discards incoming events, + drop_older discards the oldest stored events. Leave blank to use the value from + logstash.yml. + title: dead_letter_queue.storage_policy + options: + - '' + - 'drop_newer' + - 'drop_older' + advanced: True + global: False + helpLink: logstash + dead_letter_queue_x_retain_x_age: + description: >- + How long an event is kept in the dead letter queue before removal, such as 5d. Leave blank + to use the value from logstash.yml. + title: dead_letter_queue.retain.age + regex: '^$|^[0-9]+[dhms]$' + regexFailureMessage: Must be blank, or a number followed by d, h, m, or s, such as 5d. + advanced: True + global: False + helpLink: logstash + path_x_queue: + description: >- + Directory inside the Logstash container holding the persistent queue for this pipeline. The + default lives under the /nsm/logstash bind mount; a path outside it will not survive a + container restart. Leave blank to use the value from logstash.yml. + title: path.queue + advanced: True + global: False + helpLink: logstash + path_x_dead_letter_queue: + description: >- + Directory inside the Logstash container holding the dead letter queue for this pipeline. The + default lives under the /nsm/logstash bind mount; a path outside it will not survive a + container restart. Leave blank to use the value from logstash.yml. + title: path.dead_letter_queue + advanced: True + global: False + helpLink: logstash + config_x_debug: + description: >- + Whether the fully compiled configuration for this pipeline is written to the log. The output + may contain sensitive values from the pipeline configuration. Leave blank to use the value + from logstash.yml. + title: config.debug + options: + - '' + - 'true' + - 'false' + advanced: True + global: False + helpLink: logstash + config_x_support_escapes: + description: >- + Whether escape sequences such as \n and \t in this pipeline's quoted strings are + interpreted. Leave blank to use the value from logstash.yml. + title: config.support_escapes + options: + - '' + - 'true' + - 'false' + advanced: True + global: False + helpLink: logstash + fleet: *pipeline_settings + receiver: *pipeline_settings + search: *pipeline_settings + custom0: *pipeline_settings + custom1: *pipeline_settings + custom2: *pipeline_settings + custom3: *pipeline_settings + custom4: *pipeline_settings settings: lsheap: description: Heap size to use for logstash From 356da0039509ee4e256a74d8d917c06de875e593 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 20 Aug 2026 13:29:38 -0400 Subject: [PATCH 2/5] Ignore malformed logstash pipeline_settings instead of failing the state A non-mapping value under logstash:pipeline_settings: made config.sls raise "'str object' has no attribute 'get'", which failed the whole logstash.config render rather than just skipping the bad value. pipelines.yml.jinja already guarded this; config.sls now does too, and logs which pipeline was ignored. --- salt/logstash/config.sls | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/logstash/config.sls b/salt/logstash/config.sls index a661e96cc..1c5b6e9e2 100644 --- a/salt/logstash/config.sls +++ b/salt/logstash/config.sls @@ -82,7 +82,11 @@ ls_custom_pipeline_conf_{{assigned_pipeline}}_{{pipeline}}: {% for assigned_pipeline in ASSIGNED_PIPELINES %} {# a blank per-pipeline setting falls back to the global logstash.yml value #} -{% set PIPELINE_OVERRIDES = LOGSTASH_MERGED.get('pipeline_settings', {}).get(assigned_pipeline, {}) %} +{% set PARSED_OVERRIDES = LOGSTASH_MERGED.get('pipeline_settings', {}).get(assigned_pipeline, {}) %} +{% if PARSED_OVERRIDES is not mapping %} +{% do salt.log.warning('logstash: ignoring malformed pipeline_settings for pipeline ' ~ assigned_pipeline ~ '; expected a set of settings') %} +{% endif %} +{% set PIPELINE_OVERRIDES = PARSED_OVERRIDES if PARSED_OVERRIDES is mapping else {} %} {% set THREADS = PIPELINE_OVERRIDES.get('pipeline_x_workers') or LOGSTASH_MERGED.config.pipeline_x_workers %} {% set BATCH = PIPELINE_OVERRIDES.get('pipeline_x_batch_x_size') or LOGSTASH_MERGED.config.pipeline_x_batch_x_size %} {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %} From c1f256e63098503c9db36d9fc3cffd41617374ea Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 20 Aug 2026 16:35:31 -0400 Subject: [PATCH 3/5] Correct pipeline_settings annotations against Logstash 9.3.7 Widen the byte-size regex, which rejected values Logstash accepts and so blocked the save in SOC: bare-letter units (1g, 512m, 64k), decimals (1.5gb), whitespace before the unit, and a bare integer. Allow whitespace in dead_letter_queue.retain.age (5 d). Both stay lowercase-only, matching byte_value.rb and AbstractPipelineExt.parseToDuration. Fix description gaps: queue.checkpoint.retry is a Windows/SAN workaround Elastic does not otherwise recommend, batch metrics sampling is technical preview, queue.checkpoint.interval is deprecated in 9.1, compression makes a queue unreadable by Logstash before 9.2, flush_check_interval has a 1000ms floor, max_events counts unread events, and the path settings are created by Logstash but reject symlinks. Note which settings apply only to persisted queues or an enabled DLQ. Drop the undocumented 'disabled' value from queue.compression. Numeric fields stay stricter than NumericSetting, which has no validator and would accept negatives, floats and NaN in event counts and intervals. --- salt/logstash/soc_logstash.yaml | 111 +++++++++++++++++++------------- 1 file changed, 66 insertions(+), 45 deletions(-) diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index 9dfad556b..bce0e97c7 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -88,8 +88,10 @@ logstash: helpLink: logstash pipeline_x_batch_x_metrics_x_sampling_mode: description: >- - How much batch size metering this pipeline records. Fuller sampling helps size batches but - consumes additional heap. Leave blank to use the value from logstash.yml. + Controls how often batch size metrics are collected for this pipeline, which helps tune + pipeline.batch.size to the batch sizes actually being processed. Fuller sampling consumes + additional heap. Elastic marks this setting as a technical preview that may change in a + future release. Leave blank to use the value from logstash.yml. title: pipeline.batch.metrics.sampling_mode options: - '' @@ -115,8 +117,9 @@ logstash: pipeline_x_ecs_compatibility: description: >- Elastic Common Schema compatibility mode for plugins in this pipeline. Security Onion sets - this globally and it should rarely be changed per pipeline. Leave blank to use the value - from logstash.yml. + this globally and it should rarely be changed per pipeline. Elastic considers values other + than disabled to be BETA, and they may produce unintended consequences when upgrading + Logstash. Leave blank to use the value from logstash.yml. title: pipeline.ecs_compatibility options: - '' @@ -153,29 +156,32 @@ logstash: helpLink: logstash queue_x_max_bytes: description: >- - Total size of the persistent queue for this pipeline. Only applies when queue.type is - persisted, and must fit the disk backing /nsm/logstash. Leave blank to use the value from - logstash.yml. + Total capacity of the persistent queue for this pipeline, in bytes. Only applies when + queue.type is persisted. The disk backing /nsm/logstash must be larger than this value. If + both queue.max_events and queue.max_bytes are set, whichever is reached first applies. Leave + blank to use the value from logstash.yml. title: queue.max_bytes - regex: '^$|^[0-9]+(b|kb|mb|gb|tb|pb)$' - regexFailureMessage: Must be blank, or a size such as 512mb, 1gb. + regex: '^$|^[0-9]+$|^[0-9]+(\.[0-9]+)?\s*(b|kb?|mb?|gb?|tb?|pb?)$' + regexFailureMessage: Must be blank, or a size such as 512mb, 1gb, or 64k. Units are lowercase. advanced: True global: False helpLink: logstash queue_x_page_capacity: description: >- - Size of each page in the persistent queue for this pipeline. Leave blank to use the value - from logstash.yml. + Size of the individual append-only page data files that make up the persistent queue for + this pipeline. Only applies when queue.type is persisted. Leave blank to use the value from + logstash.yml. title: queue.page_capacity - regex: '^$|^[0-9]+(b|kb|mb|gb|tb|pb)$' - regexFailureMessage: Must be blank, or a size such as 512mb, 1gb. + regex: '^$|^[0-9]+$|^[0-9]+(\.[0-9]+)?\s*(b|kb?|mb?|gb?|tb?|pb?)$' + regexFailureMessage: Must be blank, or a size such as 512mb, 1gb, or 64k. Units are lowercase. advanced: True global: False helpLink: logstash queue_x_max_events: description: >- - Maximum number of events in the persistent queue for this pipeline. 0 means unlimited. Leave - blank to use the value from logstash.yml. + Maximum number of unread events in the persistent queue for this pipeline. 0 means + unlimited. Only applies when queue.type is persisted. Leave blank to use the value from + logstash.yml. title: queue.max_events regex: '^$|^[0-9]+$' regexFailureMessage: Must be blank, or a whole number. @@ -184,8 +190,8 @@ logstash: helpLink: logstash queue_x_checkpoint_x_acks: description: >- - Number of acknowledged events before a persistent queue checkpoint is forced. 0 means - unlimited. Leave blank to use the value from logstash.yml. + Maximum number of acknowledged events before a checkpoint is forced. 0 means unlimited. Only + applies when queue.type is persisted. Leave blank to use the value from logstash.yml. title: queue.checkpoint.acks regex: '^$|^[0-9]+$' regexFailureMessage: Must be blank, or a whole number. @@ -194,8 +200,9 @@ logstash: helpLink: logstash queue_x_checkpoint_x_writes: description: >- - Number of written events before a persistent queue checkpoint is forced. 0 means unlimited. - Leave blank to use the value from logstash.yml. + Maximum number of written events before a checkpoint is forced. Setting this to 1 gives + maximum durability at a severe performance cost. 0 means unlimited. Only applies when + queue.type is persisted. Leave blank to use the value from logstash.yml. title: queue.checkpoint.writes regex: '^$|^[0-9]+$' regexFailureMessage: Must be blank, or a whole number. @@ -204,8 +211,9 @@ logstash: helpLink: logstash queue_x_checkpoint_x_interval: description: >- - Milliseconds between persistent queue head page checkpoints. 0 disables periodic - checkpointing. Leave blank to use the value from logstash.yml. + Milliseconds between forced checkpoints on the persistent queue head page. 0 eliminates + periodic checkpoints. Deprecated by Elastic as of Logstash 9.1. Only applies when queue.type + is persisted. Leave blank to use the value from logstash.yml. title: queue.checkpoint.interval regex: '^$|^[0-9]+$' regexFailureMessage: Must be blank, or a whole number. @@ -214,8 +222,11 @@ logstash: helpLink: logstash queue_x_checkpoint_x_retry: description: >- - Whether Logstash retries a failed persistent queue checkpoint write. Leave blank to use the - value from logstash.yml. + When enabled, Logstash retries four times per attempted checkpoint write that fails; later + errors are not retried. Elastic describes this as a workaround for failed checkpoint writes + seen only on Windows and on filesystems with non-standard behaviour such as SANs, and does + not recommend enabling it otherwise. Only applies when queue.type is persisted. Leave blank + to use the value from logstash.yml. title: queue.checkpoint.retry options: - '' @@ -226,8 +237,11 @@ logstash: helpLink: logstash queue_x_compression: description: >- - Compression applied to persistent queue pages for this pipeline, trading CPU for disk. Leave - blank to use the value from logstash.yml. + Compression applied to persistent queue pages for this pipeline, trading CPU for disk: speed + favours the fastest operation, size the smallest files, and balanced sits between them. Once + compressed events have been written, that queue cannot be read by Logstash releases earlier + than 9.2. Only applies when queue.type is persisted. Leave blank to use the value from + logstash.yml. title: queue.compression options: - '' @@ -235,15 +249,14 @@ logstash: - 'speed' - 'balanced' - 'size' - - 'disabled' advanced: True global: False helpLink: logstash queue_x_drain: description: >- - Whether Logstash drains the persistent queue before shutting down this pipeline. Draining a - large queue makes shutdown take considerably longer. Leave blank to use the value from - logstash.yml. + When enabled, Logstash waits for the persistent queue to drain before shutting down this + pipeline. Draining a large queue makes shutdown take considerably longer. Only applies when + queue.type is persisted. Leave blank to use the value from logstash.yml. title: queue.drain options: - '' @@ -266,18 +279,20 @@ logstash: helpLink: logstash dead_letter_queue_x_max_bytes: description: >- - Total size of the dead letter queue for this pipeline. Leave blank to use the value from - logstash.yml. + Total capacity of the dead letter queue for this pipeline, in bytes. Only applies when + dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml. title: dead_letter_queue.max_bytes - regex: '^$|^[0-9]+(b|kb|mb|gb|tb|pb)$' - regexFailureMessage: Must be blank, or a size such as 512mb, 1gb. + regex: '^$|^[0-9]+$|^[0-9]+(\.[0-9]+)?\s*(b|kb?|mb?|gb?|tb?|pb?)$' + regexFailureMessage: Must be blank, or a size such as 512mb, 1gb, or 64k. Units are lowercase. advanced: True global: False helpLink: logstash dead_letter_queue_x_flush_interval: description: >- - Milliseconds before a partial dead letter queue segment is flushed. Leave blank to use the - value from logstash.yml. + Milliseconds before an incomplete dead letter queue segment is flushed and made available to + the dead_letter_queue input. Lower values write more, smaller segment files; higher values + add latency before events can be read. Only applies when dead_letter_queue.enable is true. + Leave blank to use the value from logstash.yml. title: dead_letter_queue.flush_interval regex: '^$|^[0-9]+$' regexFailureMessage: Must be blank, or a whole number. @@ -286,8 +301,9 @@ logstash: helpLink: logstash dead_letter_queue_x_flush_check_interval: description: >- - Milliseconds between checks for a dead letter queue segment that needs flushing. Leave blank - to use the value from logstash.yml. + Milliseconds between checks for a stale dead letter queue segment needing a flush. Cannot be + set lower than 1000. Smaller values rotate segments sooner at the cost of CPU. Only applies + when dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml. title: dead_letter_queue.flush_check_interval regex: '^$|^[0-9]+$' regexFailureMessage: Must be blank, or a whole number. @@ -296,9 +312,9 @@ logstash: helpLink: logstash dead_letter_queue_x_storage_policy: description: >- - What happens when the dead letter queue is full: drop_newer discards incoming events, - drop_older discards the oldest stored events. Leave blank to use the value from - logstash.yml. + Action taken when dead_letter_queue.max_bytes is reached: drop_newer stops accepting new + events, drop_older removes the oldest events to make room. Only applies when + dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml. title: dead_letter_queue.storage_policy options: - '' @@ -309,10 +325,11 @@ logstash: helpLink: logstash dead_letter_queue_x_retain_x_age: description: >- - How long an event is kept in the dead letter queue before removal, such as 5d. Leave blank - to use the value from logstash.yml. + How long an event is kept in the dead letter queue before Logstash removes it, such as 5d. + Units are d, h, m and s; there is no default unit, so one must be given. Only applies when + dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml. title: dead_letter_queue.retain.age - regex: '^$|^[0-9]+[dhms]$' + regex: '^$|^[0-9]+\s*[dhms]$' regexFailureMessage: Must be blank, or a number followed by d, h, m, or s, such as 5d. advanced: True global: False @@ -321,7 +338,9 @@ logstash: description: >- Directory inside the Logstash container holding the persistent queue for this pipeline. The default lives under the /nsm/logstash bind mount; a path outside it will not survive a - container restart. Leave blank to use the value from logstash.yml. + container restart. Logstash creates the directory if it is missing, requires it to be + writable, and refuses to start if the path is a symlink. Only applies when queue.type is + persisted. Leave blank to use the value from logstash.yml. title: path.queue advanced: True global: False @@ -330,7 +349,9 @@ logstash: description: >- Directory inside the Logstash container holding the dead letter queue for this pipeline. The default lives under the /nsm/logstash bind mount; a path outside it will not survive a - container restart. Leave blank to use the value from logstash.yml. + container restart. Logstash creates the directory if it is missing, requires it to be + writable, and refuses to start if the path is a symlink. Only applies when + dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml. title: path.dead_letter_queue advanced: True global: False From 5c3a69d74239ab4246a26be9a56f4a0c7b4493bf Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 20 Aug 2026 17:33:34 -0400 Subject: [PATCH 4/5] Warn about two pipeline_settings combinations that stop a pipeline Grid testing every permitted value on the manager pipeline surfaced two combinations the UI allows that take the pipeline down, neither of which the descriptions mentioned. pipeline.ordered: true requires pipeline.workers: 1; with more workers the pipeline fails to start with "enabling the 'pipeline.ordered' setting requires the use of a single pipeline worker". Also correct the auto wording: it only engages when workers is explicitly set to 1. queue.max_bytes larger than the free space on /nsm/logstash fails queue creation with "Unable to allocate N more bytes", rather than merely being inadvisable. --- salt/logstash/soc_logstash.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index bce0e97c7..729b8c242 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -103,8 +103,10 @@ logstash: helpLink: logstash pipeline_x_ordered: description: >- - Whether event order is preserved through this pipeline. auto enables ordering only when the - pipeline runs a single worker. Leave blank to use the value from logstash.yml. + Whether event order is preserved through this pipeline. auto enables ordering only when + pipeline.workers is explicitly set to 1, and does nothing otherwise. Setting this to true + requires pipeline.workers to be 1 as well; with more workers this pipeline fails to start. + Leave blank to use the value from logstash.yml. title: pipeline.ordered options: - '' @@ -157,8 +159,9 @@ logstash: queue_x_max_bytes: description: >- Total capacity of the persistent queue for this pipeline, in bytes. Only applies when - queue.type is persisted. The disk backing /nsm/logstash must be larger than this value. If - both queue.max_events and queue.max_bytes are set, whichever is reached first applies. Leave + queue.type is persisted. The disk backing /nsm/logstash must have room for this much data or + the pipeline fails to start, reporting that it was unable to allocate the space. If both + queue.max_events and queue.max_bytes are set, whichever is reached first applies. Leave blank to use the value from logstash.yml. title: queue.max_bytes regex: '^$|^[0-9]+$|^[0-9]+(\.[0-9]+)?\s*(b|kb?|mb?|gb?|tb?|pb?)$' From 12744353fb95e83e7939003793d03efd70761d70 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Fri, 21 Aug 2026 10:18:13 -0400 Subject: [PATCH 5/5] Allow ten custom logstash pipelines instead of five --- salt/logstash/defaults.yaml | 145 ++++++++++++++++++++++++++++++++ salt/logstash/soc_logstash.yaml | 10 +++ 2 files changed, 155 insertions(+) diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index 84bb91afd..fcaa2ed09 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -42,6 +42,11 @@ logstash: custom2: [] custom3: [] custom4: [] + custom5: [] + custom6: [] + custom7: [] + custom8: [] + custom9: [] pipeline_config: custom001: |- filter { @@ -313,6 +318,146 @@ logstash: path_x_dead_letter_queue: '' config_x_debug: '' config_x_support_escapes: '' + custom5: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' + custom6: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' + custom7: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' + custom8: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' + custom9: + pipeline_x_workers: '' + pipeline_x_batch_x_size: '' + pipeline_x_batch_x_delay: '' + pipeline_x_batch_x_metrics_x_sampling_mode: '' + pipeline_x_ordered: '' + pipeline_x_ecs_compatibility: '' + pipeline_x_reloadable: '' + queue_x_type: '' + queue_x_max_bytes: '' + queue_x_page_capacity: '' + queue_x_max_events: '' + queue_x_checkpoint_x_acks: '' + queue_x_checkpoint_x_writes: '' + queue_x_checkpoint_x_interval: '' + queue_x_checkpoint_x_retry: '' + queue_x_compression: '' + queue_x_drain: '' + dead_letter_queue_x_enable: '' + dead_letter_queue_x_max_bytes: '' + dead_letter_queue_x_flush_interval: '' + dead_letter_queue_x_flush_check_interval: '' + dead_letter_queue_x_storage_policy: '' + dead_letter_queue_x_retain_x_age: '' + path_x_queue: '' + path_x_dead_letter_queue: '' + config_x_debug: '' + config_x_support_escapes: '' settings: lsheap: 500m config: diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index 729b8c242..ea037e58a 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -35,6 +35,11 @@ logstash: custom2: *defined_pipelines custom3: *defined_pipelines custom4: *defined_pipelines + custom5: *defined_pipelines + custom6: *defined_pipelines + custom7: *defined_pipelines + custom8: *defined_pipelines + custom9: *defined_pipelines pipeline_config: custom001: &pipeline_config description: Pipeline configuration for Logstash @@ -392,6 +397,11 @@ logstash: custom2: *pipeline_settings custom3: *pipeline_settings custom4: *pipeline_settings + custom5: *pipeline_settings + custom6: *pipeline_settings + custom7: *pipeline_settings + custom8: *pipeline_settings + custom9: *pipeline_settings settings: lsheap: description: Heap size to use for logstash