Expose Logstash 9.3.7 pipeline settings per pipeline in SOC

Add logstash:pipeline_settings carrying the 27 pipeline-scoped settings
Logstash 9.3.7 accepts, annotated individually per pipeline and rendered
into pipelines.yml. A blank setting inherits from logstash.yml. Restart
logstash when pipelines.yml changes, and add the missing managerhype
annotation.

Fixes #15090

VERSION

@@ -1 +1 @@
-3.0.0-foxtrot
+3.3.0

salt/elasticfleet/integration-defaults.map.jinja

@@ -16,6 +16,7 @@
     'awsfirehose.metrics': 'aws.cloudwatch',
     'cribl.logs': 'cribl',
     'cribl.metrics': 'cribl',
+    'sentinel_one_cloud_funnel.logins': 'sentinel_one_cloud_funnel.login',
     'azure_application_insights.app_insights': 'azure.app_insights',
     'azure_application_insights.app_state': 'azure.app_state',
     'azure_billing.billing': 'azure.billing',

salt/elasticsearch/cluster.sls

@@ -98,15 +98,6 @@ so-es-cluster-settings:
       - docker_container: so-elasticsearch
       - file: elasticsearch_sbin_jinja
       - http: wait_for_so-elasticsearch
-      
-so-elasticsearch-system-indices-patch:
-  cmd.run:
-    - name: /usr/sbin/so-elasticsearch-system-indices-patch
-    - require:
-      - http: wait_for_so-elasticsearch
-      - file: so-elasticsearch-system-indices-patch-script
-    - onchanges:
-      - file: so-elasticsearch-system-indices-patch-script
 {%  endif %}
 
 # heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)

salt/elasticsearch/config.sls

@@ -42,16 +42,6 @@ elasticsearch_sbin:
     - file_mode: 755
     - exclude_pat:
       - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state
-      - so-elasticsearch-system-indices-patch
-    - show_changes: False
-
-so-elasticsearch-system-indices-patch-script:
-  file.managed:
-    - name: /usr/sbin/so-elasticsearch-system-indices-patch
-    - source: salt://elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch
-    - user: 930
-    - group: 939
-    - mode: 755
     - show_changes: False
 
 elasticsearch_sbin_jinja:

salt/elasticsearch/defaults.yaml

@@ -1,7 +1,7 @@
 elasticsearch:
   enabled: false
   esheap: '600m'
-  version: 9.4.5
+  version: 9.3.7
   index_clean: true
   data_retention_method: DLM
   vm:

salt/elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch

@@ -1,123 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-set -eo pipefail
-
-SETTINGS='{"index":{"auto_expand_replicas":"0-1"}}'
-KIBANA_PASSWORD=
-INDEX_PATTERNS=(
-    '.entity_analytics.risk_score.lookup-*'
-    '.entity_analytics.watchlists.*'
-    '.workflows-executions'
-    '.workflows-step-executions'
-    '.entities.v2.latest.security_*'
-    '.entities.v2.history.security_*'
-    'risk-score.risk-score-latest-*'
-)
-DATA_STREAM_PATTERNS=(
-    '.entities.v2.updates.security_*'
-    'risk-score.risk-score-*'
-)
-TEMPLATE_PATTERNS=(
-    'entities_v2_latest_security_default_index_template'
-    'entities_v2_history_security_default_index_template'
-    '.entities_v2_updates_security_default_index_template'
-    '.risk-score.risk-score-default-index-template'
-)
-
-query_es() {
-    if so-elasticsearch-query "$@" --fail --retry 3 --retry-delay 5; then
-        return 0
-    fi
-    
-    # retry failed attempts with so_kibana user (system managed indices reject so_elastic user)
-    local query_path="$1"
-    shift
-
-    if [[ -z "$KIBANA_PASSWORD" ]]; then
-        KIBANA_PASSWORD=$(salt-call pillar.get elasticsearch:auth:users:so_kibana_user:pass --out=newline_values_only)
-    fi
-    [[ -n "$KIBANA_PASSWORD" ]] || return 1
-
-    echo "Retrying ${query_path} as so_kibana." >&2
-    curl -K /opt/so/conf/elasticsearch/curl.config --user "so_kibana:${KIBANA_PASSWORD}" \
-        -s -k -L --fail --retry 3 --retry-delay 5 -H 'Content-Type: application/json' "https://localhost:9200/${query_path}" "$@"
-}
-
-# add auto_expand_replicas=0-1 to given index
-set_auto_expand_replicas() {
-    local index="$1"
-
-    echo "Setting auto_expand_replicas to 0-1 on ${index}."
-    query_es "${index}/_settings" -XPUT -d "$SETTINGS" >/dev/null
-}
-
-# resolve index patterns and run set_auto_expand_replicas on each index
-update_system_indices() {
-    local pattern="$1"
-    local response index
-
-    if ! response=$(query_es "_resolve/index/${pattern}?expand_wildcards=all" 2>/dev/null); then
-        return 0
-    fi
-
-    while read -r index; do
-        [[ -n "$index" ]] && set_auto_expand_replicas "$index"
-    done < <(jq -r '.indices[]?.name' <<<"$response")
-}
-
-# get data stream backing indices and run set_auto_expand_replicas on each index
-update_system_ds() {
-    local pattern="$1"
-    local response index
-
-    if ! response=$(query_es "_data_stream/${pattern}?expand_wildcards=all" 2>/dev/null); then
-        return 0
-    fi
-    # find backing indices for each data stream and update with $SETTINGS
-    while read -r index; do
-        [[ -n "$index" ]] && set_auto_expand_replicas "$index"
-    done < <(jq -r '.data_streams[]?.indices[]?.index_name' <<<"$response")
-}
-
-# get index templates, update with auto_expand_replicas=0-1, and PUT back. Keeping mappings/settings/aliases in-place
-update_system_templates() {
-    local pattern="$1"
-    local templates name response template auto_expand_replicas
-
-    if ! templates=$(query_es "_index_template/${pattern}" 2>/dev/null); then
-        return 0
-    fi
-    while read -r name; do
-        response=$(query_es "_index_template/${name}")
-        template=$(jq -c '.index_templates[0].index_template' <<<"$response")
-        auto_expand_replicas=$(jq -r '.template.settings["index.auto_expand_replicas"] // .template.settings.index.auto_expand_replicas // empty' <<<"$template")
-        [[ "$auto_expand_replicas" == "0-1" ]] && continue
-
-        template=$(jq '
-            if (.template.settings.index | type) == "object" then
-                .template.settings.index.auto_expand_replicas = "0-1"
-            else
-                .template.settings["index.auto_expand_replicas"] = "0-1"
-            end
-            | del(.created_date_millis, .modified_date_millis)
-        ' <<<"$template")
-        echo "Setting auto_expand_replicas to 0-1 on index template ${name}."
-        query_es "_index_template/${name}" -XPUT -d "$template" >/dev/null
-    done < <(jq -r '.index_templates[]?.name' <<<"$templates")
-}
-
-for pattern in "${INDEX_PATTERNS[@]}"; do
-    update_system_indices "$pattern"
-done
-
-for pattern in "${DATA_STREAM_PATTERNS[@]}"; do
-    update_system_ds "$pattern"
-done
-
-for pattern in "${TEMPLATE_PATTERNS[@]}"; do
-    update_system_templates "$pattern"
-done

salt/logstash/config.sls

@@ -81,6 +81,10 @@ ls_custom_pipeline_conf_{{assigned_pipeline}}_{{pipeline}}:
 
 
 {% for assigned_pipeline in ASSIGNED_PIPELINES %}
+{# a blank per-pipeline setting falls back to the global logstash.yml value #}
+{% set PIPELINE_OVERRIDES = LOGSTASH_MERGED.get('pipeline_settings', {}).get(assigned_pipeline, {}) %}
+{% set THREADS = PIPELINE_OVERRIDES.get('pipeline_x_workers') or LOGSTASH_MERGED.config.pipeline_x_workers %}
+{% set BATCH = PIPELINE_OVERRIDES.get('pipeline_x_batch_x_size') or LOGSTASH_MERGED.config.pipeline_x_batch_x_size %}
     {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %}
 ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}:
   file.managed:
@@ -92,8 +96,8 @@ ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_")
         GLOBALS: {{ GLOBALS }}
         ES_USER: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') }}"
         ES_PASS: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') }}"
-        THREADS: {{ LOGSTASH_MERGED.config.pipeline_x_workers }}
-        BATCH: {{ LOGSTASH_MERGED.config.pipeline_x_batch_x_size }}
+        THREADS: {{ THREADS }}
+        BATCH: {{ BATCH }}
       {% else %}
     - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{CONFIGFILE.split('/')[1]}}
       {% endif %}

salt/logstash/defaults.yaml

@@ -60,6 +60,259 @@ logstash:
     custom008: PLACEHOLDER
     custom009: PLACEHOLDER
     custom010: PLACEHOLDER
+  pipeline_settings:
+    fleet:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    manager:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    receiver:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    search:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom0:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom1:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom2:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom3:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
+    custom4:
+      pipeline_x_workers: ''
+      pipeline_x_batch_x_size: ''
+      pipeline_x_batch_x_delay: ''
+      pipeline_x_batch_x_metrics_x_sampling_mode: ''
+      pipeline_x_ordered: ''
+      pipeline_x_ecs_compatibility: ''
+      pipeline_x_reloadable: ''
+      queue_x_type: ''
+      queue_x_max_bytes: ''
+      queue_x_page_capacity: ''
+      queue_x_max_events: ''
+      queue_x_checkpoint_x_acks: ''
+      queue_x_checkpoint_x_writes: ''
+      queue_x_checkpoint_x_interval: ''
+      queue_x_checkpoint_x_retry: ''
+      queue_x_compression: ''
+      queue_x_drain: ''
+      dead_letter_queue_x_enable: ''
+      dead_letter_queue_x_max_bytes: ''
+      dead_letter_queue_x_flush_interval: ''
+      dead_letter_queue_x_flush_check_interval: ''
+      dead_letter_queue_x_storage_policy: ''
+      dead_letter_queue_x_retain_x_age: ''
+      path_x_queue: ''
+      path_x_dead_letter_queue: ''
+      config_x_debug: ''
+      config_x_support_escapes: ''
   settings:
     lsheap: 500m
   config:

salt/logstash/enabled.sls

@@ -105,6 +105,7 @@ so-logstash:
     {% endif %}
     - watch:
       - file: lsetcsync
+      - file: lspipelinesyml
       - file: trusttheca
       {% if GLOBALS.is_manager %}
       - file: elasticsearch_cacerts

salt/logstash/etc/pipelines.yml.jinja

@@ -1,4 +1,17 @@
+{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
+{%- set PIPELINE_SETTINGS = LOGSTASH_MERGED.get('pipeline_settings', {}) %}
 {%- for assigned_pipeline in ASSIGNED_PIPELINES %}
 - pipeline.id: {{ assigned_pipeline }}
   path.config: "/usr/share/logstash/pipelines/{{ assigned_pipeline }}/"
+{%-   set extra = PIPELINE_SETTINGS.get(assigned_pipeline, {}) %}
+{%-   if extra is mapping %}
+{#-     values are emitted unquoted so yaml re-infers the type logstash expects:
+        4 as an integer, false as a boolean, 1024mb and auto as strings #}
+{%-     for key, value in extra | dictsort %}
+{%-       set rendered = key | replace('_x_', '.') %}
+{%-       if value not in ['', None] and rendered not in ['pipeline.id', 'path.config'] %}
+  {{ rendered }}: {{ value }}
+{%-       endif %}
+{%-     endfor %}
+{%-   endif %}
 {% endfor -%}

salt/logstash/soc_logstash.yaml

@@ -16,6 +16,7 @@ logstash:
       heavynode: *assigned_pipelines
       searchnode: *assigned_pipelines
       manager: *assigned_pipelines
+      managerhype: *assigned_pipelines
       managersearch: *assigned_pipelines
       fleet: *assigned_pipelines
   defined_pipelines:
@@ -51,6 +52,322 @@ logstash:
     custom008: *pipeline_config
     custom009: *pipeline_config
     custom010: *pipeline_config
+  pipeline_settings:
+    manager: &pipeline_settings
+      pipeline_x_workers:
+        description: >-
+          Number of worker threads that run filters and outputs for this pipeline. May be set higher
+          than the CPU core count when outputs spend time waiting on I/O. Leave blank to use the value
+          from logstash.yml.
+        title: pipeline.workers
+        regex: '^$|^[1-9][0-9]*$'
+        regexFailureMessage: Must be blank, or a positive whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      pipeline_x_batch_x_size:
+        description: >-
+          Maximum number of events an individual worker thread collects before running filters and
+          outputs. Larger batches are more efficient but increase heap use; total in-flight events is
+          workers multiplied by batch size. Leave blank to use the value from logstash.yml.
+        title: pipeline.batch.size
+        regex: '^$|^[1-9][0-9]*$'
+        regexFailureMessage: Must be blank, or a positive whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      pipeline_x_batch_x_delay:
+        description: >-
+          Milliseconds a worker waits for the next event before running a batch that is not yet full.
+          Leave blank to use the value from logstash.yml.
+        title: pipeline.batch.delay
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      pipeline_x_batch_x_metrics_x_sampling_mode:
+        description: >-
+          How much batch size metering this pipeline records. Fuller sampling helps size batches but
+          consumes additional heap. Leave blank to use the value from logstash.yml.
+        title: pipeline.batch.metrics.sampling_mode
+        options:
+        - ''
+        - 'disabled'
+        - 'minimal'
+        - 'full'
+        advanced: True
+        global: False
+        helpLink: logstash
+      pipeline_x_ordered:
+        description: >-
+          Whether event order is preserved through this pipeline. auto enables ordering only when the
+          pipeline runs a single worker. Leave blank to use the value from logstash.yml.
+        title: pipeline.ordered
+        options:
+        - ''
+        - 'auto'
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+      pipeline_x_ecs_compatibility:
+        description: >-
+          Elastic Common Schema compatibility mode for plugins in this pipeline. Security Onion sets
+          this globally and it should rarely be changed per pipeline. Leave blank to use the value
+          from logstash.yml.
+        title: pipeline.ecs_compatibility
+        options:
+        - ''
+        - 'disabled'
+        - 'v1'
+        - 'v8'
+        advanced: True
+        global: False
+        helpLink: logstash
+      pipeline_x_reloadable:
+        description: >-
+          Whether this pipeline may be reloaded when its configuration changes. Leave blank to use the
+          value from logstash.yml.
+        title: pipeline.reloadable
+        options:
+        - ''
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_type:
+        description: >-
+          Queue backing this pipeline. persisted buffers events to disk under /nsm/logstash so they
+          survive a restart, at some throughput cost; memory does not. Leave blank to use the value
+          from logstash.yml.
+        title: queue.type
+        options:
+        - ''
+        - 'memory'
+        - 'persisted'
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_max_bytes:
+        description: >-
+          Total size of the persistent queue for this pipeline. Only applies when queue.type is
+          persisted, and must fit the disk backing /nsm/logstash. Leave blank to use the value from
+          logstash.yml.
+        title: queue.max_bytes
+        regex: '^$|^[0-9]+(b|kb|mb|gb|tb|pb)$'
+        regexFailureMessage: Must be blank, or a size such as 512mb, 1gb.
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_page_capacity:
+        description: >-
+          Size of each page in the persistent queue for this pipeline. Leave blank to use the value
+          from logstash.yml.
+        title: queue.page_capacity
+        regex: '^$|^[0-9]+(b|kb|mb|gb|tb|pb)$'
+        regexFailureMessage: Must be blank, or a size such as 512mb, 1gb.
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_max_events:
+        description: >-
+          Maximum number of events in the persistent queue for this pipeline. 0 means unlimited. Leave
+          blank to use the value from logstash.yml.
+        title: queue.max_events
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_checkpoint_x_acks:
+        description: >-
+          Number of acknowledged events before a persistent queue checkpoint is forced. 0 means
+          unlimited. Leave blank to use the value from logstash.yml.
+        title: queue.checkpoint.acks
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_checkpoint_x_writes:
+        description: >-
+          Number of written events before a persistent queue checkpoint is forced. 0 means unlimited.
+          Leave blank to use the value from logstash.yml.
+        title: queue.checkpoint.writes
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_checkpoint_x_interval:
+        description: >-
+          Milliseconds between persistent queue head page checkpoints. 0 disables periodic
+          checkpointing. Leave blank to use the value from logstash.yml.
+        title: queue.checkpoint.interval
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_checkpoint_x_retry:
+        description: >-
+          Whether Logstash retries a failed persistent queue checkpoint write. Leave blank to use the
+          value from logstash.yml.
+        title: queue.checkpoint.retry
+        options:
+        - ''
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_compression:
+        description: >-
+          Compression applied to persistent queue pages for this pipeline, trading CPU for disk. Leave
+          blank to use the value from logstash.yml.
+        title: queue.compression
+        options:
+        - ''
+        - 'none'
+        - 'speed'
+        - 'balanced'
+        - 'size'
+        - 'disabled'
+        advanced: True
+        global: False
+        helpLink: logstash
+      queue_x_drain:
+        description: >-
+          Whether Logstash drains the persistent queue before shutting down this pipeline. Draining a
+          large queue makes shutdown take considerably longer. Leave blank to use the value from
+          logstash.yml.
+        title: queue.drain
+        options:
+        - ''
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+      dead_letter_queue_x_enable:
+        description: >-
+          Whether events this pipeline cannot process are written to a dead letter queue instead of
+          being dropped. Leave blank to use the value from logstash.yml.
+        title: dead_letter_queue.enable
+        options:
+        - ''
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+      dead_letter_queue_x_max_bytes:
+        description: >-
+          Total size of the dead letter queue for this pipeline. Leave blank to use the value from
+          logstash.yml.
+        title: dead_letter_queue.max_bytes
+        regex: '^$|^[0-9]+(b|kb|mb|gb|tb|pb)$'
+        regexFailureMessage: Must be blank, or a size such as 512mb, 1gb.
+        advanced: True
+        global: False
+        helpLink: logstash
+      dead_letter_queue_x_flush_interval:
+        description: >-
+          Milliseconds before a partial dead letter queue segment is flushed. Leave blank to use the
+          value from logstash.yml.
+        title: dead_letter_queue.flush_interval
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      dead_letter_queue_x_flush_check_interval:
+        description: >-
+          Milliseconds between checks for a dead letter queue segment that needs flushing. Leave blank
+          to use the value from logstash.yml.
+        title: dead_letter_queue.flush_check_interval
+        regex: '^$|^[0-9]+$'
+        regexFailureMessage: Must be blank, or a whole number.
+        advanced: True
+        global: False
+        helpLink: logstash
+      dead_letter_queue_x_storage_policy:
+        description: >-
+          What happens when the dead letter queue is full: drop_newer discards incoming events,
+          drop_older discards the oldest stored events. Leave blank to use the value from
+          logstash.yml.
+        title: dead_letter_queue.storage_policy
+        options:
+        - ''
+        - 'drop_newer'
+        - 'drop_older'
+        advanced: True
+        global: False
+        helpLink: logstash
+      dead_letter_queue_x_retain_x_age:
+        description: >-
+          How long an event is kept in the dead letter queue before removal, such as 5d. Leave blank
+          to use the value from logstash.yml.
+        title: dead_letter_queue.retain.age
+        regex: '^$|^[0-9]+[dhms]$'
+        regexFailureMessage: Must be blank, or a number followed by d, h, m, or s, such as 5d.
+        advanced: True
+        global: False
+        helpLink: logstash
+      path_x_queue:
+        description: >-
+          Directory inside the Logstash container holding the persistent queue for this pipeline. The
+          default lives under the /nsm/logstash bind mount; a path outside it will not survive a
+          container restart. Leave blank to use the value from logstash.yml.
+        title: path.queue
+        advanced: True
+        global: False
+        helpLink: logstash
+      path_x_dead_letter_queue:
+        description: >-
+          Directory inside the Logstash container holding the dead letter queue for this pipeline. The
+          default lives under the /nsm/logstash bind mount; a path outside it will not survive a
+          container restart. Leave blank to use the value from logstash.yml.
+        title: path.dead_letter_queue
+        advanced: True
+        global: False
+        helpLink: logstash
+      config_x_debug:
+        description: >-
+          Whether the fully compiled configuration for this pipeline is written to the log. The output
+          may contain sensitive values from the pipeline configuration. Leave blank to use the value
+          from logstash.yml.
+        title: config.debug
+        options:
+        - ''
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+      config_x_support_escapes:
+        description: >-
+          Whether escape sequences such as \n and \t in this pipeline's quoted strings are
+          interpreted. Leave blank to use the value from logstash.yml.
+        title: config.support_escapes
+        options:
+        - ''
+        - 'true'
+        - 'false'
+        advanced: True
+        global: False
+        helpLink: logstash
+    fleet: *pipeline_settings
+    receiver: *pipeline_settings
+    search: *pipeline_settings
+    custom0: *pipeline_settings
+    custom1: *pipeline_settings
+    custom2: *pipeline_settings
+    custom3: *pipeline_settings
+    custom4: *pipeline_settings
   settings:
     lsheap: 
       description: Heap size to use for logstash