Merge pull request #16176 from Security-Onion-Solutions/3/dev

3/dev

VERSION

@@ -1 +1 @@
-3.3.0
+3.0.0-foxtrot

salt/elasticfleet/integration-defaults.map.jinja

@@ -16,7 +16,6 @@
     'awsfirehose.metrics': 'aws.cloudwatch',
     'cribl.logs': 'cribl',
     'cribl.metrics': 'cribl',
-    'sentinel_one_cloud_funnel.logins': 'sentinel_one_cloud_funnel.login',
     'azure_application_insights.app_insights': 'azure.app_insights',
     'azure_application_insights.app_state': 'azure.app_state',
     'azure_billing.billing': 'azure.billing',

salt/elasticsearch/cluster.sls

@@ -98,6 +98,13 @@ so-es-cluster-settings:
       - docker_container: so-elasticsearch
       - file: elasticsearch_sbin_jinja
       - http: wait_for_so-elasticsearch
+
+so-elasticsearch-system-indices-patch:
+  cmd.run:
+    - name: /usr/sbin/so-elasticsearch-system-indices-patch
+    - require:
+      - http: wait_for_so-elasticsearch
+      - file: so-elasticsearch-system-indices-patch-script
 {%  endif %}
 
 # heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)

salt/elasticsearch/config.sls

@@ -42,6 +42,16 @@ elasticsearch_sbin:
     - file_mode: 755
     - exclude_pat:
       - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state
+      - so-elasticsearch-system-indices-patch
+    - show_changes: False
+
+so-elasticsearch-system-indices-patch-script:
+  file.managed:
+    - name: /usr/sbin/so-elasticsearch-system-indices-patch
+    - source: salt://elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch
+    - user: 930
+    - group: 939
+    - mode: 755
     - show_changes: False
 
 elasticsearch_sbin_jinja:

salt/elasticsearch/defaults.yaml

@@ -1,7 +1,7 @@
 elasticsearch:
   enabled: false
   esheap: '600m'
-  version: 9.3.7
+  version: 9.4.5
   index_clean: true
   data_retention_method: DLM
   vm:

salt/elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch

@@ -0,0 +1,187 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+set -eo pipefail
+
+SETTINGS='{"index":{"auto_expand_replicas":"0-1"}}'
+KIBANA_PASSWORD=
+INDEX_PATTERNS=(
+  '.entity_analytics.risk_score.lookup-*'
+  '.entity_analytics.watchlists.*'
+  '.workflows-executions'
+  '.workflows-step-executions'
+  '.entities.v2.latest.security_*'
+  '.entities.v2.history.security_*'
+  'risk-score.risk-score-latest-*'
+)
+DATA_STREAM_PATTERNS=(
+  '.entities.v2.updates.security_*'
+  'risk-score.risk-score-*'
+)
+TEMPLATE_PATTERNS=(
+  'entities_v2_latest_security_default_index_template'
+  'entities_v2_history_security_default_index_template'
+  '.entities_v2_updates_security_default_index_template'
+  '.risk-score.risk-score-default-index-template'
+)
+
+query_es() {
+  if so-elasticsearch-query "$@" --fail --retry 3 --retry-delay 5; then
+    return 0
+  fi
+
+  # retry failed attempts with so_kibana user (system managed indices reject so_elastic user)
+  local query_path="$1"
+  shift
+
+  if [[ -z "$KIBANA_PASSWORD" ]]; then
+    KIBANA_PASSWORD=$(salt-call pillar.get elasticsearch:auth:users:so_kibana_user:pass --out=newline_values_only)
+  fi
+  [[ -n "$KIBANA_PASSWORD" ]] || return 1
+
+  echo "Retrying ${query_path} as so_kibana." >&2
+  curl -K /opt/so/conf/elasticsearch/curl.config --user "so_kibana:${KIBANA_PASSWORD}" \
+    -s -k -L --fail --retry 3 --retry-delay 5 -H 'Content-Type: application/json' "https://localhost:9200/${query_path}" "$@"
+}
+
+# add auto_expand_replicas=0-1 to given index
+set_auto_expand_replicas() {
+  local index="$1"
+
+  echo "Setting auto_expand_replicas to 0-1 on ${index}."
+  query_es "${index}/_settings" -XPUT -d "$SETTINGS" >/dev/null
+}
+
+# resolve index patterns and find each index with an unassigned replica
+unassigned_replicas() {
+  local pattern="$1"
+  local resolved_indices response index
+
+  if ! resolved_indices=$(query_es "_resolve/index/${pattern}?expand_wildcards=all" 2>/dev/null); then
+    return 0
+  fi
+
+  while read -r index; do
+    if ! response=$(query_es "_cat/shards/${index}?format=json&h=index,prirep,state" 2>/dev/null); then
+      continue
+    fi
+    jq -r '.[]? | objects | select(.prirep == "r" and .state == "UNASSIGNED") | .index' <<<"$response"
+  done < <(jq -r '.indices[]?.name' <<<"$resolved_indices")
+}
+
+data_stream_indices() {
+  local pattern="$1"
+  local response
+
+  if ! response=$(query_es "_data_stream/${pattern}?expand_wildcards=all" 2>/dev/null); then
+    return 0
+  fi
+  jq -r '.data_streams[]?.indices[]?.index_name' <<<"$response"
+}
+
+update_system_indices() {
+  local pattern="$1"
+  local index
+
+  while read -r index; do
+    [[ -n "$index" ]] && set_auto_expand_replicas "$index"
+  done < <(unassigned_replicas "$pattern")
+}
+
+# update data stream backing indices with unassigned replicas
+update_system_ds() {
+  local pattern="$1"
+  local index
+
+  while read -r index; do
+    while read -r unassigned_index; do
+      [[ -n "$unassigned_index" ]] && set_auto_expand_replicas "$unassigned_index"
+    done < <(unassigned_replicas "$index")
+  done < <(data_stream_indices "$pattern")
+}
+
+has_unassigned_replicas() {
+  local pattern="$1"
+  local index
+
+  index=$(unassigned_replicas "$pattern" | sed -n '1p')
+  [[ -n "$index" ]]
+}
+
+data_stream_has_unassigned_replicas() {
+  local pattern="$1"
+  local index
+  while read -r index; do
+    has_unassigned_replicas "$index" && return 0
+  done < <(data_stream_indices "$pattern")
+
+  return 1
+}
+
+needs_patch() {
+  local pattern
+  for pattern in "${INDEX_PATTERNS[@]}"; do
+    has_unassigned_replicas "$pattern" && return 0
+  done
+  for pattern in "${DATA_STREAM_PATTERNS[@]}"; do
+    data_stream_has_unassigned_replicas "$pattern" && return 0
+  done
+
+  return 1
+}
+
+# get index templates, update with auto_expand_replicas=0-1, and PUT back. Keeping mappings/settings/aliases in-place
+update_system_templates() {
+  local pattern="$1"
+  local templates name response template auto_expand_replicas
+
+  if ! templates=$(query_es "_index_template/${pattern}" 2>/dev/null); then
+    return 0
+  fi
+  while read -r name; do
+    response=$(query_es "_index_template/${name}")
+    template=$(jq -c '.index_templates[0].index_template' <<<"$response")
+    auto_expand_replicas=$(jq -r '.template.settings["index.auto_expand_replicas"] // .template.settings.index.auto_expand_replicas // empty' <<<"$template")
+    [[ "$auto_expand_replicas" == "0-1" ]] && continue
+
+    template=$(jq '
+            if (.template.settings.index | type) == "object" then
+                .template.settings.index.auto_expand_replicas = "0-1"
+            else
+                .template.settings["index.auto_expand_replicas"] = "0-1"
+            end
+            | del(.created_date_millis, .modified_date_millis)
+        ' <<<"$template")
+    echo "Setting auto_expand_replicas to 0-1 on index template ${name}."
+    query_es "_index_template/${name}" -XPUT -d "$template" >/dev/null
+  done < <(jq -r '.index_templates[]?.name' <<<"$templates")
+}
+
+if [[ "${1:-}" == "--check" ]]; then
+  needs_patch
+  exit $?
+fi
+
+if [[ $# -ne 0 ]]; then
+  echo "Usage: $0 [--check]" >&2
+  exit 1
+fi
+
+for pattern in "${INDEX_PATTERNS[@]}"; do
+  if has_unassigned_replicas "$pattern"; then
+    update_system_indices "$pattern"
+  fi
+done
+
+for pattern in "${DATA_STREAM_PATTERNS[@]}"; do
+  if data_stream_has_unassigned_replicas "$pattern"; then
+    update_system_ds "$pattern"
+  fi
+done
+
+for pattern in "${TEMPLATE_PATTERNS[@]}"; do
+  update_system_templates "$pattern"
+done

salt/logstash/config.sls

@@ -81,14 +81,6 @@ ls_custom_pipeline_conf_{{assigned_pipeline}}_{{pipeline}}:
 
 
 {% for assigned_pipeline in ASSIGNED_PIPELINES %}
-{# a blank per-pipeline setting falls back to the global logstash.yml value #}
-{% set PARSED_OVERRIDES = LOGSTASH_MERGED.get('pipeline_settings', {}).get(assigned_pipeline, {}) %}
-{% if PARSED_OVERRIDES is not mapping %}
-{%   do salt.log.warning('logstash: ignoring malformed pipeline_settings for pipeline ' ~ assigned_pipeline ~ '; expected a set of settings') %}
-{% endif %}
-{% set PIPELINE_OVERRIDES = PARSED_OVERRIDES if PARSED_OVERRIDES is mapping else {} %}
-{% set THREADS = PIPELINE_OVERRIDES.get('pipeline_x_workers') or LOGSTASH_MERGED.config.pipeline_x_workers %}
-{% set BATCH = PIPELINE_OVERRIDES.get('pipeline_x_batch_x_size') or LOGSTASH_MERGED.config.pipeline_x_batch_x_size %}
     {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %}
 ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}:
   file.managed:
@@ -100,8 +92,8 @@ ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_")
         GLOBALS: {{ GLOBALS }}
         ES_USER: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') }}"
         ES_PASS: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') }}"
-        THREADS: {{ THREADS }}
-        BATCH: {{ BATCH }}
+        THREADS: {{ LOGSTASH_MERGED.config.pipeline_x_workers }}
+        BATCH: {{ LOGSTASH_MERGED.config.pipeline_x_batch_x_size }}
       {% else %}
     - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{CONFIGFILE.split('/')[1]}}
       {% endif %}

salt/logstash/defaults.yaml

@@ -60,259 +60,6 @@ logstash:
     custom008: PLACEHOLDER
     custom009: PLACEHOLDER
     custom010: PLACEHOLDER
-  pipeline_settings:
-    fleet:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    manager:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    receiver:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    search:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom0:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom1:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom2:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom3:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
-    custom4:
-      pipeline_x_workers: ''
-      pipeline_x_batch_x_size: ''
-      pipeline_x_batch_x_delay: ''
-      pipeline_x_batch_x_metrics_x_sampling_mode: ''
-      pipeline_x_ordered: ''
-      pipeline_x_ecs_compatibility: ''
-      pipeline_x_reloadable: ''
-      queue_x_type: ''
-      queue_x_max_bytes: ''
-      queue_x_page_capacity: ''
-      queue_x_max_events: ''
-      queue_x_checkpoint_x_acks: ''
-      queue_x_checkpoint_x_writes: ''
-      queue_x_checkpoint_x_interval: ''
-      queue_x_checkpoint_x_retry: ''
-      queue_x_compression: ''
-      queue_x_drain: ''
-      dead_letter_queue_x_enable: ''
-      dead_letter_queue_x_max_bytes: ''
-      dead_letter_queue_x_flush_interval: ''
-      dead_letter_queue_x_flush_check_interval: ''
-      dead_letter_queue_x_storage_policy: ''
-      dead_letter_queue_x_retain_x_age: ''
-      path_x_queue: ''
-      path_x_dead_letter_queue: ''
-      config_x_debug: ''
-      config_x_support_escapes: ''
   settings:
     lsheap: 500m
   config:

salt/logstash/enabled.sls

@@ -105,7 +105,6 @@ so-logstash:
     {% endif %}
     - watch:
       - file: lsetcsync
-      - file: lspipelinesyml
       - file: trusttheca
       {% if GLOBALS.is_manager %}
       - file: elasticsearch_cacerts

salt/logstash/etc/pipelines.yml.jinja

@@ -1,17 +1,4 @@
-{%- from 'logstash/map.jinja' import LOGSTASH_MERGED %}
-{%- set PIPELINE_SETTINGS = LOGSTASH_MERGED.get('pipeline_settings', {}) %}
 {%- for assigned_pipeline in ASSIGNED_PIPELINES %}
 - pipeline.id: {{ assigned_pipeline }}
   path.config: "/usr/share/logstash/pipelines/{{ assigned_pipeline }}/"
-{%-   set extra = PIPELINE_SETTINGS.get(assigned_pipeline, {}) %}
-{%-   if extra is mapping %}
-{#-     values are emitted unquoted so yaml re-infers the type logstash expects:
-        4 as an integer, false as a boolean, 1024mb and auto as strings #}
-{%-     for key, value in extra | dictsort %}
-{%-       set rendered = key | replace('_x_', '.') %}
-{%-       if value not in ['', None] and rendered not in ['pipeline.id', 'path.config'] %}
-  {{ rendered }}: {{ value }}
-{%-       endif %}
-{%-     endfor %}
-{%-   endif %}
 {% endfor -%}

salt/logstash/soc_logstash.yaml

@@ -16,7 +16,6 @@ logstash:
       heavynode: *assigned_pipelines
       searchnode: *assigned_pipelines
       manager: *assigned_pipelines
-      managerhype: *assigned_pipelines
       managersearch: *assigned_pipelines
       fleet: *assigned_pipelines
   defined_pipelines:
@@ -52,343 +51,6 @@ logstash:
     custom008: *pipeline_config
     custom009: *pipeline_config
     custom010: *pipeline_config
-  pipeline_settings:
-    manager: &pipeline_settings
-      pipeline_x_workers:
-        description: >-
-          Number of worker threads that run filters and outputs for this pipeline. May be set higher
-          than the CPU core count when outputs spend time waiting on I/O. Leave blank to use the value
-          from logstash.yml.
-        title: pipeline.workers
-        regex: '^$|^[1-9][0-9]*$'
-        regexFailureMessage: Must be blank, or a positive whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      pipeline_x_batch_x_size:
-        description: >-
-          Maximum number of events an individual worker thread collects before running filters and
-          outputs. Larger batches are more efficient but increase heap use; total in-flight events is
-          workers multiplied by batch size. Leave blank to use the value from logstash.yml.
-        title: pipeline.batch.size
-        regex: '^$|^[1-9][0-9]*$'
-        regexFailureMessage: Must be blank, or a positive whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      pipeline_x_batch_x_delay:
-        description: >-
-          Milliseconds a worker waits for the next event before running a batch that is not yet full.
-          Leave blank to use the value from logstash.yml.
-        title: pipeline.batch.delay
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      pipeline_x_batch_x_metrics_x_sampling_mode:
-        description: >-
-          Controls how often batch size metrics are collected for this pipeline, which helps tune
-          pipeline.batch.size to the batch sizes actually being processed. Fuller sampling consumes
-          additional heap. Elastic marks this setting as a technical preview that may change in a
-          future release. Leave blank to use the value from logstash.yml.
-        title: pipeline.batch.metrics.sampling_mode
-        options:
-        - ''
-        - 'disabled'
-        - 'minimal'
-        - 'full'
-        advanced: True
-        global: False
-        helpLink: logstash
-      pipeline_x_ordered:
-        description: >-
-          Whether event order is preserved through this pipeline. auto enables ordering only when the
-          pipeline runs a single worker. Leave blank to use the value from logstash.yml.
-        title: pipeline.ordered
-        options:
-        - ''
-        - 'auto'
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-      pipeline_x_ecs_compatibility:
-        description: >-
-          Elastic Common Schema compatibility mode for plugins in this pipeline. Security Onion sets
-          this globally and it should rarely be changed per pipeline. Elastic considers values other
-          than disabled to be BETA, and they may produce unintended consequences when upgrading
-          Logstash. Leave blank to use the value from logstash.yml.
-        title: pipeline.ecs_compatibility
-        options:
-        - ''
-        - 'disabled'
-        - 'v1'
-        - 'v8'
-        advanced: True
-        global: False
-        helpLink: logstash
-      pipeline_x_reloadable:
-        description: >-
-          Whether this pipeline may be reloaded when its configuration changes. Leave blank to use the
-          value from logstash.yml.
-        title: pipeline.reloadable
-        options:
-        - ''
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_type:
-        description: >-
-          Queue backing this pipeline. persisted buffers events to disk under /nsm/logstash so they
-          survive a restart, at some throughput cost; memory does not. Leave blank to use the value
-          from logstash.yml.
-        title: queue.type
-        options:
-        - ''
-        - 'memory'
-        - 'persisted'
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_max_bytes:
-        description: >-
-          Total capacity of the persistent queue for this pipeline, in bytes. Only applies when
-          queue.type is persisted. The disk backing /nsm/logstash must be larger than this value. If
-          both queue.max_events and queue.max_bytes are set, whichever is reached first applies. Leave
-          blank to use the value from logstash.yml.
-        title: queue.max_bytes
-        regex: '^$|^[0-9]+$|^[0-9]+(\.[0-9]+)?\s*(b|kb?|mb?|gb?|tb?|pb?)$'
-        regexFailureMessage: Must be blank, or a size such as 512mb, 1gb, or 64k. Units are lowercase.
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_page_capacity:
-        description: >-
-          Size of the individual append-only page data files that make up the persistent queue for
-          this pipeline. Only applies when queue.type is persisted. Leave blank to use the value from
-          logstash.yml.
-        title: queue.page_capacity
-        regex: '^$|^[0-9]+$|^[0-9]+(\.[0-9]+)?\s*(b|kb?|mb?|gb?|tb?|pb?)$'
-        regexFailureMessage: Must be blank, or a size such as 512mb, 1gb, or 64k. Units are lowercase.
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_max_events:
-        description: >-
-          Maximum number of unread events in the persistent queue for this pipeline. 0 means
-          unlimited. Only applies when queue.type is persisted. Leave blank to use the value from
-          logstash.yml.
-        title: queue.max_events
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_checkpoint_x_acks:
-        description: >-
-          Maximum number of acknowledged events before a checkpoint is forced. 0 means unlimited. Only
-          applies when queue.type is persisted. Leave blank to use the value from logstash.yml.
-        title: queue.checkpoint.acks
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_checkpoint_x_writes:
-        description: >-
-          Maximum number of written events before a checkpoint is forced. Setting this to 1 gives
-          maximum durability at a severe performance cost. 0 means unlimited. Only applies when
-          queue.type is persisted. Leave blank to use the value from logstash.yml.
-        title: queue.checkpoint.writes
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_checkpoint_x_interval:
-        description: >-
-          Milliseconds between forced checkpoints on the persistent queue head page. 0 eliminates
-          periodic checkpoints. Deprecated by Elastic as of Logstash 9.1. Only applies when queue.type
-          is persisted. Leave blank to use the value from logstash.yml.
-        title: queue.checkpoint.interval
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_checkpoint_x_retry:
-        description: >-
-          When enabled, Logstash retries four times per attempted checkpoint write that fails; later
-          errors are not retried. Elastic describes this as a workaround for failed checkpoint writes
-          seen only on Windows and on filesystems with non-standard behaviour such as SANs, and does
-          not recommend enabling it otherwise. Only applies when queue.type is persisted. Leave blank
-          to use the value from logstash.yml.
-        title: queue.checkpoint.retry
-        options:
-        - ''
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_compression:
-        description: >-
-          Compression applied to persistent queue pages for this pipeline, trading CPU for disk: speed
-          favours the fastest operation, size the smallest files, and balanced sits between them. Once
-          compressed events have been written, that queue cannot be read by Logstash releases earlier
-          than 9.2. Only applies when queue.type is persisted. Leave blank to use the value from
-          logstash.yml.
-        title: queue.compression
-        options:
-        - ''
-        - 'none'
-        - 'speed'
-        - 'balanced'
-        - 'size'
-        advanced: True
-        global: False
-        helpLink: logstash
-      queue_x_drain:
-        description: >-
-          When enabled, Logstash waits for the persistent queue to drain before shutting down this
-          pipeline. Draining a large queue makes shutdown take considerably longer. Only applies when
-          queue.type is persisted. Leave blank to use the value from logstash.yml.
-        title: queue.drain
-        options:
-        - ''
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-      dead_letter_queue_x_enable:
-        description: >-
-          Whether events this pipeline cannot process are written to a dead letter queue instead of
-          being dropped. Leave blank to use the value from logstash.yml.
-        title: dead_letter_queue.enable
-        options:
-        - ''
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-      dead_letter_queue_x_max_bytes:
-        description: >-
-          Total capacity of the dead letter queue for this pipeline, in bytes. Only applies when
-          dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml.
-        title: dead_letter_queue.max_bytes
-        regex: '^$|^[0-9]+$|^[0-9]+(\.[0-9]+)?\s*(b|kb?|mb?|gb?|tb?|pb?)$'
-        regexFailureMessage: Must be blank, or a size such as 512mb, 1gb, or 64k. Units are lowercase.
-        advanced: True
-        global: False
-        helpLink: logstash
-      dead_letter_queue_x_flush_interval:
-        description: >-
-          Milliseconds before an incomplete dead letter queue segment is flushed and made available to
-          the dead_letter_queue input. Lower values write more, smaller segment files; higher values
-          add latency before events can be read. Only applies when dead_letter_queue.enable is true.
-          Leave blank to use the value from logstash.yml.
-        title: dead_letter_queue.flush_interval
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      dead_letter_queue_x_flush_check_interval:
-        description: >-
-          Milliseconds between checks for a stale dead letter queue segment needing a flush. Cannot be
-          set lower than 1000. Smaller values rotate segments sooner at the cost of CPU. Only applies
-          when dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml.
-        title: dead_letter_queue.flush_check_interval
-        regex: '^$|^[0-9]+$'
-        regexFailureMessage: Must be blank, or a whole number.
-        advanced: True
-        global: False
-        helpLink: logstash
-      dead_letter_queue_x_storage_policy:
-        description: >-
-          Action taken when dead_letter_queue.max_bytes is reached: drop_newer stops accepting new
-          events, drop_older removes the oldest events to make room. Only applies when
-          dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml.
-        title: dead_letter_queue.storage_policy
-        options:
-        - ''
-        - 'drop_newer'
-        - 'drop_older'
-        advanced: True
-        global: False
-        helpLink: logstash
-      dead_letter_queue_x_retain_x_age:
-        description: >-
-          How long an event is kept in the dead letter queue before Logstash removes it, such as 5d.
-          Units are d, h, m and s; there is no default unit, so one must be given. Only applies when
-          dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml.
-        title: dead_letter_queue.retain.age
-        regex: '^$|^[0-9]+\s*[dhms]$'
-        regexFailureMessage: Must be blank, or a number followed by d, h, m, or s, such as 5d.
-        advanced: True
-        global: False
-        helpLink: logstash
-      path_x_queue:
-        description: >-
-          Directory inside the Logstash container holding the persistent queue for this pipeline. The
-          default lives under the /nsm/logstash bind mount; a path outside it will not survive a
-          container restart. Logstash creates the directory if it is missing, requires it to be
-          writable, and refuses to start if the path is a symlink. Only applies when queue.type is
-          persisted. Leave blank to use the value from logstash.yml.
-        title: path.queue
-        advanced: True
-        global: False
-        helpLink: logstash
-      path_x_dead_letter_queue:
-        description: >-
-          Directory inside the Logstash container holding the dead letter queue for this pipeline. The
-          default lives under the /nsm/logstash bind mount; a path outside it will not survive a
-          container restart. Logstash creates the directory if it is missing, requires it to be
-          writable, and refuses to start if the path is a symlink. Only applies when
-          dead_letter_queue.enable is true. Leave blank to use the value from logstash.yml.
-        title: path.dead_letter_queue
-        advanced: True
-        global: False
-        helpLink: logstash
-      config_x_debug:
-        description: >-
-          Whether the fully compiled configuration for this pipeline is written to the log. The output
-          may contain sensitive values from the pipeline configuration. Leave blank to use the value
-          from logstash.yml.
-        title: config.debug
-        options:
-        - ''
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-      config_x_support_escapes:
-        description: >-
-          Whether escape sequences such as \n and \t in this pipeline's quoted strings are
-          interpreted. Leave blank to use the value from logstash.yml.
-        title: config.support_escapes
-        options:
-        - ''
-        - 'true'
-        - 'false'
-        advanced: True
-        global: False
-        helpLink: logstash
-    fleet: *pipeline_settings
-    receiver: *pipeline_settings
-    search: *pipeline_settings
-    custom0: *pipeline_settings
-    custom1: *pipeline_settings
-    custom2: *pipeline_settings
-    custom3: *pipeline_settings
-    custom4: *pipeline_settings
   settings:
     lsheap: 
       description: Heap size to use for logstash

salt/soc/defaults.yaml

@@ -2671,7 +2671,7 @@ soc:
               # The id (UUIDv4) is pregenerated and can safely be used.
               # Click "Convert" to convert the Sigma rule to use Security Onion field mappings within an EQL query
               #
-              # Rule Creation Guide: https://github.com/SigmaHQ/sigma/wiki/Rule-Creation-Guide
+              # Rule Creation Guide: https://github.com/SigmaHQ/sigma/wiki/Rule-Creation-High%E2%80%90Level-Guide
               # Logsources: https://sigmahq.io/docs/basics/log-sources.html
 
               title: 'A Short Capitalized Title With Less Than 50 Characters'
@@ -2683,7 +2683,7 @@ soc:
               references:
                 - 'https://local.invalid'
               author: '@SecurityOnion'
-              date: 'YYYY/MM/DD'
+              date: '[today]'
               tags:
                 - detection.threat_hunting
                 - attack.technique_id