From 5d0c187497a0cb24fdb1f6e3d99fea0a1482280c Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 23 Jan 2026 14:45:31 -0600 Subject: [PATCH 01/10] format json --- salt/elasticsearch/files/ingest/global@custom | 228 +++++++++++++++--- 1 file changed, 198 insertions(+), 30 deletions(-) diff --git a/salt/elasticsearch/files/ingest/global@custom b/salt/elasticsearch/files/ingest/global@custom index 8e48eb0b9..6bf36d1a3 100644 --- a/salt/elasticsearch/files/ingest/global@custom +++ b/salt/elasticsearch/files/ingest/global@custom @@ -1,31 +1,199 @@ { - "version": 3, - "_meta": { - "managed_by": "securityonion", - "managed": true - }, - "description": "Custom pipeline for processing all incoming Fleet Agent documents. \n", - "processors": [ - { "set": { "ignore_failure": true, "field": "event.module", "value": "elastic_agent" } }, - { "split": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp" } }, - { "split": { "if": "ctx.data_stream?.dataset != null && ctx.data_stream?.dataset.contains('.')", "field":"data_stream.dataset", "separator":"\\.", "target_field":"datastream_dataset_temp", "ignore_missing":true } }, - { "set": { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } }, - { "set": { "if": "ctx.datastream_dataset_temp != null && ctx.datastream_dataset_temp[0] == 'network_traffic'", "field":"event.module", "value":"{{ datastream_dataset_temp.0 }}", "ignore_failure":true, "ignore_empty_value":true, "description":"Fix EA network packet capture" } }, - { "gsub": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } }, - { "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}", "allow_duplicates": false } }, - { "set": { "if": "ctx.network?.direction == 'egress'", "override": true, "field": "network.initiated", "value": "true" } }, - { "set": { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } }, - { "set": { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } }, - { "set": { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } }, - { "set": { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.dataset", "value": "import" } }, - { "set": { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.namespace", "value": "so" } }, - { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true } }, - { "set": { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } }, - { "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } }, - { "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } }, - { "set": { "if": "ctx.event?.dataset != null && ctx.event?.dataset == 'elasticsearch.server'", "field": "event.module", "value":"elasticsearch" }}, - {"append": {"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"if":"ctx?.event?.dataset == 'endpoint.events.network' && ctx?.source?.ip != null","ignore_failure":true}}, - {"foreach": {"field":"host.ip","processor":{"append":{"field":"related.ip","value":"{{_ingest._value}}","allow_duplicates":false}},"if":"ctx?.event?.module == 'endpoint' && ctx?.host?.ip != null","ignore_missing":true, "description":"Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"}}, - { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp", "datastream_dataset_temp" ], "ignore_missing": true, "ignore_failure": true } } - ] -} + "version": 3, + "_meta": { + "managed_by": "securityonion", + "managed": true + }, + "description": "Custom pipeline for processing all incoming Fleet Agent documents. \n", + "processors": [ + { + "set": { + "ignore_failure": true, + "field": "event.module", + "value": "elastic_agent" + } + }, + { + "split": { + "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", + "field": "event.dataset", + "separator": "\\.", + "target_field": "module_temp" + } + }, + { + "split": { + "if": "ctx.data_stream?.dataset != null && ctx.data_stream?.dataset.contains('.')", + "field": "data_stream.dataset", + "separator": "\\.", + "target_field": "datastream_dataset_temp", + "ignore_missing": true + } + }, + { + "set": { + "if": "ctx.module_temp != null", + "override": true, + "field": "event.module", + "value": "{{module_temp.0}}" + } + }, + { + "set": { + "if": "ctx.datastream_dataset_temp != null && ctx.datastream_dataset_temp[0] == 'network_traffic'", + "field": "event.module", + "value": "{{ datastream_dataset_temp.0 }}", + "ignore_failure": true, + "ignore_empty_value": true, + "description": "Fix EA network packet capture" + } + }, + { + "gsub": { + "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", + "field": "event.dataset", + "pattern": "^[^.]*.", + "replacement": "", + "target_field": "dataset_tag_temp" + } + }, + { + "append": { + "if": "ctx.dataset_tag_temp != null", + "field": "tags", + "value": "{{dataset_tag_temp}}", + "allow_duplicates": false + } + }, + { + "set": { + "if": "ctx.network?.direction == 'egress'", + "override": true, + "field": "network.initiated", + "value": "true" + } + }, + { + "set": { + "if": "ctx.network?.direction == 'ingress'", + "override": true, + "field": "network.initiated", + "value": "false" + } + }, + { + "set": { + "if": "ctx.network?.type == 'ipv4'", + "override": true, + "field": "destination.ipv6", + "value": "false" + } + }, + { + "set": { + "if": "ctx.network?.type == 'ipv6'", + "override": true, + "field": "destination.ipv6", + "value": "true" + } + }, + { + "set": { + "if": "ctx.tags != null && ctx.tags.contains('import')", + "override": true, + "field": "data_stream.dataset", + "value": "import" + } + }, + { + "set": { + "if": "ctx.tags != null && ctx.tags.contains('import')", + "override": true, + "field": "data_stream.namespace", + "value": "so" + } + }, + { + "community_id": { + "if": "ctx.event?.dataset == 'endpoint.events.network'", + "ignore_failure": true + } + }, + { + "set": { + "if": "ctx.event?.module == 'fim'", + "override": true, + "field": "event.module", + "value": "file_integrity" + } + }, + { + "rename": { + "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", + "ignore_missing": true, + "field": "winlog.event_data.Threat Name", + "target_field": "winlog.event_data.threat_name" + } + }, + { + "set": { + "if": "ctx?.metadata?.kafka != null", + "field": "kafka.id", + "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", + "ignore_failure": true + } + }, + { + "set": { + "if": "ctx.event?.dataset != null && ctx.event?.dataset == 'elasticsearch.server'", + "field": "event.module", + "value": "elasticsearch" + } + }, + { + "append": { + "field": "related.ip", + "value": [ + "{{source.ip}}", + "{{destination.ip}}" + ], + "allow_duplicates": false, + "if": "ctx?.event?.dataset == 'endpoint.events.network' && ctx?.source?.ip != null", + "ignore_failure": true + } + }, + { + "foreach": { + "field": "host.ip", + "processor": { + "append": { + "field": "related.ip", + "value": "{{_ingest._value}}", + "allow_duplicates": false + } + }, + "if": "ctx?.event?.module == 'endpoint' && ctx?.host?.ip != null", + "ignore_missing": true, + "description": "Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip" + } + }, + { + "remove": { + "field": [ + "message2", + "type", + "fields", + "category", + "module", + "dataset", + "event.dataset_temp", + "dataset_tag_temp", + "module_temp", + "datastream_dataset_temp" + ], + "ignore_missing": true, + "ignore_failure": true + } + } + ] +} \ No newline at end of file From 32f030f6f602847b1801bfb80b79350f184241fa Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 26 Jan 2026 12:24:31 -0600 Subject: [PATCH 02/10] formatting --- .../sbin/so-elastic-fleet-integration-policy-load | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load index ca260891f..2cd9401d1 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load @@ -17,9 +17,9 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then # Third, configure Elastic Defend Integration seperately /usr/sbin/so-elastic-fleet-integration-policy-elastic-defend + # Initial Endpoints - for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/endpoints-initial/*.json - do + for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/endpoints-initial/*.json; do printf "\n\nInitial Endpoints Policy - Loading $INTEGRATION\n" elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION" if [ -n "$INTEGRATION_ID" ]; then @@ -40,8 +40,7 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then done # Grid Nodes - General - for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_general/*.json - do + for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_general/*.json; do printf "\n\nGrid Nodes Policy_General - Loading $INTEGRATION\n" elastic_fleet_integration_check "so-grid-nodes_general" "$INTEGRATION" if [ -n "$INTEGRATION_ID" ]; then @@ -65,8 +64,7 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then fi # Grid Nodes - Heavy - for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy/*.json - do + for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy/*.json; do printf "\n\nGrid Nodes Policy_Heavy - Loading $INTEGRATION\n" elastic_fleet_integration_check "so-grid-nodes_heavy" "$INTEGRATION" if [ -n "$INTEGRATION_ID" ]; then @@ -92,8 +90,7 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then fi # Fleet Server - Optional integrations - for INTEGRATION in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json - do + for INTEGRATION in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json; do if ! [ "$INTEGRATION" == "/opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json" ]; then FLEET_POLICY=`echo "$INTEGRATION"| cut -d'/' -f7` printf "\n\nFleet Server Policy - Loading $INTEGRATION\n" @@ -117,6 +114,7 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then fi fi done + if [[ "$RETURN_CODE" != "1" ]]; then touch /opt/so/state/eaintegrations.txt fi From a78e0b08714b91fd78b5a4b20f5bc2e5040f74ae Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 26 Jan 2026 12:26:21 -0600 Subject: [PATCH 03/10] only create /opt/so/state/eaintegrations.txt when all policies have been created/updated successfully --- .../tools/sbin/so-elastic-fleet-integration-policy-load | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load index 2cd9401d1..b8adfb76c 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load @@ -59,9 +59,6 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then fi fi done - if [[ "$RETURN_CODE" != "1" ]]; then - touch /opt/so/state/eaintegrations.txt - fi # Grid Nodes - Heavy for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy/*.json; do @@ -85,9 +82,6 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then fi fi done - if [[ "$RETURN_CODE" != "1" ]]; then - touch /opt/so/state/eaintegrations.txt - fi # Fleet Server - Optional integrations for INTEGRATION in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json; do @@ -115,6 +109,7 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then fi done + # Only create the state file if all policies were created/updated successfully if [[ "$RETURN_CODE" != "1" ]]; then touch /opt/so/state/eaintegrations.txt fi From 8cf0d59560f01ff16db24477888a35abbb4e7765 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 26 Jan 2026 12:48:15 -0600 Subject: [PATCH 04/10] remove block of elasticsearch-logs integration on heavynodes --- .../sbin/so-elastic-fleet-integration-policy-load | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load index b8adfb76c..e548c7f86 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load @@ -73,12 +73,10 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then fi else printf "\n\nIntegration does not exist - Creating integration\n" - if [ "$NAME" != "elasticsearch-logs" ]; then - if ! elastic_fleet_integration_create "@$INTEGRATION"; then - echo -e "\nFailed to create integration for ${INTEGRATION##*/}" - RETURN_CODE=1 - continue - fi + if ! elastic_fleet_integration_create "@$INTEGRATION"; then + echo -e "\nFailed to create integration for ${INTEGRATION##*/}" + RETURN_CODE=1 + continue fi fi done From 8900f9ade33e705357609470aa0e24cbeac77b27 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 26 Jan 2026 13:51:58 -0600 Subject: [PATCH 05/10] collect elasticsearch logs on heavynodes via fleet managed elastic agent --- .../elasticsearch-grid-nodes.json | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 salt/elasticfleet/files/integrations/grid-nodes_heavy/elasticsearch-grid-nodes.json diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/elasticsearch-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/elasticsearch-grid-nodes.json new file mode 100644 index 000000000..43c0c92b2 --- /dev/null +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/elasticsearch-grid-nodes.json @@ -0,0 +1,107 @@ +{ + "package": { + "name": "elasticsearch", + "version": "" + }, + "name": "elasticsearch-grid-nodes_heavy", + "namespace": "default", + "description": "Elasticsearch Logs", + "policy_id": "so-grid-nodes_heavy", + "inputs": { + "elasticsearch-logfile": { + "enabled": true, + "streams": { + "elasticsearch.audit": { + "enabled": false, + "vars": { + "paths": [ + "/var/log/elasticsearch/*_audit.json" + ] + } + }, + "elasticsearch.deprecation": { + "enabled": false, + "vars": { + "paths": [ + "/var/log/elasticsearch/*_deprecation.json" + ] + } + }, + "elasticsearch.gc": { + "enabled": false, + "vars": { + "paths": [ + "/var/log/elasticsearch/gc.log.[0-9]*", + "/var/log/elasticsearch/gc.log" + ] + } + }, + "elasticsearch.server": { + "enabled": true, + "vars": { + "paths": [ + "/opt/so/log/elasticsearch/*.json" + ] + } + }, + "elasticsearch.slowlog": { + "enabled": false, + "vars": { + "paths": [ + "/var/log/elasticsearch/*_index_search_slowlog.json", + "/var/log/elasticsearch/*_index_indexing_slowlog.json" + ] + } + } + } + }, + "elasticsearch-elasticsearch/metrics": { + "enabled": false, + "vars": { + "hosts": [ + "http://localhost:9200" + ], + "scope": "node" + }, + "streams": { + "elasticsearch.stack_monitoring.ccr": { + "enabled": false + }, + "elasticsearch.stack_monitoring.cluster_stats": { + "enabled": false + }, + "elasticsearch.stack_monitoring.enrich": { + "enabled": false + }, + "elasticsearch.stack_monitoring.index": { + "enabled": false + }, + "elasticsearch.stack_monitoring.index_recovery": { + "enabled": false, + "vars": { + "active.only": true + } + }, + "elasticsearch.stack_monitoring.index_summary": { + "enabled": false + }, + "elasticsearch.stack_monitoring.ml_job": { + "enabled": false + }, + "elasticsearch.stack_monitoring.node": { + "enabled": false + }, + "elasticsearch.stack_monitoring.node_stats": { + "enabled": false + }, + "elasticsearch.stack_monitoring.pending_tasks": { + "enabled": false + }, + "elasticsearch.stack_monitoring.shard": { + "enabled": false + } + } + } + }, + "force": true +} From 950852d673bd2565e622289bf71a74c1ca8e2945 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 26 Jan 2026 13:57:19 -0600 Subject: [PATCH 06/10] update heavynode standalone elastic agent policy --- .../files/elastic-agent.yml.jinja | 402 ++++++------------ 1 file changed, 138 insertions(+), 264 deletions(-) diff --git a/salt/elasticagent/files/elastic-agent.yml.jinja b/salt/elasticagent/files/elastic-agent.yml.jinja index 7d0b93344..ec4620efb 100644 --- a/salt/elasticagent/files/elastic-agent.yml.jinja +++ b/salt/elasticagent/files/elastic-agent.yml.jinja @@ -3,7 +3,7 @@ {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} id: aea1ba80-1065-11ee-a369-97538913b6a9 -revision: 1 +revision: 4 outputs: default: type: elasticsearch @@ -22,105 +22,9 @@ agent: metrics: false features: {} inputs: - - id: logfile-logs-fefef78c-422f-4cfa-8abf-4cd1b9428f62 - name: import-evtx-logs - revision: 2 - type: logfile - use_output: default - meta: - package: - name: log - version: - data_stream: - namespace: so - package_policy_id: fefef78c-422f-4cfa-8abf-4cd1b9428f62 - streams: - - id: logfile-log.log-fefef78c-422f-4cfa-8abf-4cd1b9428f62 - data_stream: - dataset: import - paths: - - /nsm/import/*/evtx/*.json - processors: - - dissect: - field: log.file.path - tokenizer: '/nsm/import/%{import.id}/evtx/%{import.file}' - target_prefix: '' - - decode_json_fields: - fields: - - message - target: '' - - drop_fields: - ignore_missing: true - fields: - - host - - add_fields: - fields: - dataset: system.security - type: logs - namespace: default - target: data_stream - - add_fields: - fields: - dataset: system.security - module: system - imported: true - target: event - - then: - - add_fields: - fields: - dataset: windows.sysmon_operational - target: data_stream - - add_fields: - fields: - dataset: windows.sysmon_operational - module: windows - imported: true - target: event - if: - equals: - winlog.channel: Microsoft-Windows-Sysmon/Operational - - then: - - add_fields: - fields: - dataset: system.application - target: data_stream - - add_fields: - fields: - dataset: system.application - target: event - if: - equals: - winlog.channel: Application - - then: - - add_fields: - fields: - dataset: system.system - target: data_stream - - add_fields: - fields: - dataset: system.system - target: event - if: - equals: - winlog.channel: System - - then: - - add_fields: - fields: - dataset: windows.powershell_operational - target: data_stream - - add_fields: - fields: - dataset: windows.powershell_operational - module: windows - target: event - if: - equals: - winlog.channel: Microsoft-Windows-PowerShell/Operational - tags: - - import - - id: logfile-redis-fc98c947-7d17-4861-a318-7ad075f6d1b0 + - id: logfile-redis-8b7c8390-25ef-11f0-a18d-1b26f69b8310 name: redis-logs - revision: 2 + revision: 3 type: logfile use_output: default meta: @@ -129,135 +33,147 @@ inputs: version: data_stream: namespace: default - package_policy_id: fc98c947-7d17-4861-a318-7ad075f6d1b0 + package_policy_id: 8b7c8390-25ef-11f0-a18d-1b26f69b8310 streams: - - id: logfile-redis.log-fc98c947-7d17-4861-a318-7ad075f6d1b0 + - id: logfile-redis.log-8b7c8390-25ef-11f0-a18d-1b26f69b8310 data_stream: dataset: redis.log type: logs - exclude_files: - - .gz$ paths: - - /opt/so/log/redis/redis.log + - /opt/so/log/redis/redis-server.log tags: - redis-log + exclude_files: + - .gz$ exclude_lines: - - '^\s+[\-`(''.|_]' - - id: logfile-logs-3b56803d-5ade-4c93-b25e-9b37182f66b8 + - ^\s+[\-`('.|_] + - id: filestream-filestream-85820eb0-25ef-11f0-a18d-1b26f69b8310 name: import-suricata-logs - revision: 2 - type: logfile + revision: 3 + type: filestream use_output: default meta: package: - name: log + name: filestream version: data_stream: namespace: so - package_policy_id: 3b56803d-5ade-4c93-b25e-9b37182f66b8 + package_policy_id: 85820eb0-25ef-11f0-a18d-1b26f69b8310 streams: - - id: logfile-log.log-3b56803d-5ade-4c93-b25e-9b37182f66b8 + - id: filestream-filestream.generic-85820eb0-25ef-11f0-a18d-1b26f69b8310 data_stream: dataset: import - pipeline: suricata.common paths: - /nsm/import/*/suricata/eve*.json + pipeline: suricata.common + prospector.scanner.recursive_glob: true + prospector.scanner.exclude_files: + - \.gz$ + ignore_older: 72h + clean_inactive: -1 + parsers: null processors: - add_fields: + target: event fields: + category: network module: suricata imported: true - category: network - target: event - dissect: + tokenizer: /nsm/import/%{import.id}/suricata/%{import.file} field: log.file.path - tokenizer: '/nsm/import/%{import.id}/suricata/%{import.file}' target_prefix: '' - - id: logfile-logs-c327e1a3-1ebe-449c-a8eb-f6f35032e69d - name: soc-server-logs - revision: 2 - type: logfile + file_identity.native: null + prospector.scanner.fingerprint.enabled: false + - id: filestream-filestream-86b4e960-25ef-11f0-a18d-1b26f69b8310 + name: import-zeek-logs + revision: 3 + type: filestream use_output: default meta: package: - name: log + name: filestream version: data_stream: namespace: so - package_policy_id: c327e1a3-1ebe-449c-a8eb-f6f35032e69d + package_policy_id: 86b4e960-25ef-11f0-a18d-1b26f69b8310 streams: - - id: logfile-log.log-c327e1a3-1ebe-449c-a8eb-f6f35032e69d + - id: filestream-filestream.generic-86b4e960-25ef-11f0-a18d-1b26f69b8310 data_stream: - dataset: soc - pipeline: common + dataset: import paths: - - /opt/so/log/soc/sensoroni-server.log + - /nsm/import/*/zeek/logs/*.log + prospector.scanner.recursive_glob: true + prospector.scanner.exclude_files: + - >- + (broker|capture_loss|cluster|conn-summary|console|ecat_arp_info|known_certs|known_hosts|known_services|loaded_scripts|ntp|ocsp|packet_filter|reporter|stats|stderr|stdout).log$ + clean_inactive: -1 + parsers: null processors: - - decode_json_fields: - add_error_key: true - process_array: true - max_depth: 2 - fields: - - message - target: soc + - dissect: + tokenizer: /nsm/import/%{import.id}/zeek/logs/%{import.file} + field: log.file.path + target_prefix: '' + - script: + lang: javascript + source: | + function process(event) { + var pl = event.Get("import.file").slice(0,-4); + event.Put("@metadata.pipeline", "zeek." + pl); + } - add_fields: - fields: - module: soc - dataset_temp: server - category: host target: event - - rename: - ignore_missing: true fields: - - from: soc.fields.sourceIp - to: source.ip - - from: soc.fields.status - to: http.response.status_code - - from: soc.fields.method - to: http.request.method - - from: soc.fields.path - to: url.path - - from: soc.message - to: event.action - - from: soc.level - to: log.level - tags: - - so-soc - - id: logfile-logs-906e0d4c-9ec3-4c6a-bef6-e347ec9fd073 + category: network + module: zeek + imported: true + - add_tags: + tags: ics + when: + regexp: + import.file: >- + ^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm* + file_identity.native: null + prospector.scanner.fingerprint.enabled: false + - id: filestream-filestream-91741240-25ef-11f0-a18d-1b26f69b8310 name: soc-sensoroni-logs - revision: 2 - type: logfile + revision: 3 + type: filestream use_output: default meta: package: - name: log + name: filestream version: data_stream: namespace: so - package_policy_id: 906e0d4c-9ec3-4c6a-bef6-e347ec9fd073 + package_policy_id: 91741240-25ef-11f0-a18d-1b26f69b8310 streams: - - id: logfile-log.log-906e0d4c-9ec3-4c6a-bef6-e347ec9fd073 + - id: filestream-filestream.generic-91741240-25ef-11f0-a18d-1b26f69b8310 data_stream: dataset: soc - pipeline: common paths: - /opt/so/log/sensoroni/sensoroni.log + pipeline: common + prospector.scanner.recursive_glob: true + prospector.scanner.exclude_files: + - \.gz$ + clean_inactive: -1 + parsers: null processors: - decode_json_fields: - add_error_key: true - process_array: true - max_depth: 2 fields: - message target: sensoroni + process_array: true + max_depth: 2 + add_error_key: true - add_fields: + target: event fields: + category: host module: soc dataset_temp: sensoroni - category: host - target: event - rename: - ignore_missing: true fields: - from: sensoroni.fields.sourceIp to: source.ip @@ -271,141 +187,100 @@ inputs: to: event.action - from: sensoroni.level to: log.level - - id: logfile-logs-df0d7f2c-221f-433b-b18b-d1cf83250515 - name: soc-salt-relay-logs - revision: 2 - type: logfile - use_output: default - meta: - package: - name: log - version: - data_stream: - namespace: so - package_policy_id: df0d7f2c-221f-433b-b18b-d1cf83250515 - streams: - - id: logfile-log.log-df0d7f2c-221f-433b-b18b-d1cf83250515 - data_stream: - dataset: soc - pipeline: common - paths: - - /opt/so/log/soc/salt-relay.log - processors: - - dissect: - field: message - tokenizer: '%{soc.ts} | %{event.action}' - target_prefix: '' - - add_fields: - fields: - module: soc - dataset_temp: salt_relay - category: host - target: event - tags: - - so-soc - - id: logfile-logs-74bd2366-fe52-493c-bddc-843a017fc4d0 - name: soc-auth-sync-logs - revision: 2 - type: logfile - use_output: default - meta: - package: - name: log - version: - data_stream: - namespace: so - package_policy_id: 74bd2366-fe52-493c-bddc-843a017fc4d0 - streams: - - id: logfile-log.log-74bd2366-fe52-493c-bddc-843a017fc4d0 - data_stream: - dataset: soc - pipeline: common - paths: - - /opt/so/log/soc/sync.log - processors: - - dissect: - field: message - tokenizer: '%{event.action}' - target_prefix: '' - - add_fields: - fields: - module: soc - dataset_temp: auth_sync - category: host - target: event - tags: - - so-soc - - id: logfile-logs-d151d9bf-ff2a-4529-9520-c99244bc0253 + ignore_missing: true + file_identity.native: null + prospector.scanner.fingerprint.enabled: false + - id: filestream-filestream-976e3900-25ef-11f0-a18d-1b26f69b8310 name: suricata-logs - revision: 2 - type: logfile + revision: 3 + type: filestream use_output: default meta: package: - name: log + name: filestream version: data_stream: namespace: so - package_policy_id: d151d9bf-ff2a-4529-9520-c99244bc0253 + package_policy_id: 976e3900-25ef-11f0-a18d-1b26f69b8310 streams: - - id: logfile-log.log-d151d9bf-ff2a-4529-9520-c99244bc0253 + - id: filestream-filestream.generic-976e3900-25ef-11f0-a18d-1b26f69b8310 data_stream: dataset: suricata - pipeline: suricata.common paths: - /nsm/suricata/eve*.json + pipeline: suricata.common + prospector.scanner.recursive_glob: true + prospector.scanner.exclude_files: + - \.gz$ + clean_inactive: -1 + parsers: null processors: - add_fields: - fields: - module: suricata - category: network target: event - - id: logfile-logs-31f94d05-ae75-40ee-b9c5-0e0356eff327 + fields: + category: network + module: suricata + file_identity.native: null + prospector.scanner.fingerprint.enabled: false + - id: filestream-filestream-95091fe0-25ef-11f0-a18d-1b26f69b8310 name: strelka-logs - revision: 2 - type: logfile + revision: 3 + type: filestream use_output: default meta: package: - name: log + name: filestream version: data_stream: namespace: so - package_policy_id: 31f94d05-ae75-40ee-b9c5-0e0356eff327 + package_policy_id: 95091fe0-25ef-11f0-a18d-1b26f69b8310 streams: - - id: logfile-log.log-31f94d05-ae75-40ee-b9c5-0e0356eff327 + - id: filestream-filestream.generic-95091fe0-25ef-11f0-a18d-1b26f69b8310 data_stream: dataset: strelka - pipeline: strelka.file paths: - /nsm/strelka/log/strelka.log + pipeline: strelka.file + prospector.scanner.recursive_glob: true + prospector.scanner.exclude_files: + - \.gz$ + clean_inactive: -1 + parsers: null processors: - add_fields: - fields: - module: strelka - category: file target: event - - id: logfile-logs-6197fe84-9b58-4d9b-8464-3d517f28808d + fields: + category: file + module: strelka + file_identity.native: null + prospector.scanner.fingerprint.enabled: false + - id: filestream-filestream-9f309ca0-25ef-11f0-a18d-1b26f69b8310 name: zeek-logs - revision: 1 - type: logfile + revision: 2 + type: filestream use_output: default meta: package: - name: log - version: + name: filestream + version: data_stream: namespace: so - package_policy_id: 6197fe84-9b58-4d9b-8464-3d517f28808d + package_policy_id: 9f309ca0-25ef-11f0-a18d-1b26f69b8310 streams: - - id: logfile-log.log-6197fe84-9b58-4d9b-8464-3d517f28808d + - id: filestream-filestream.generic-9f309ca0-25ef-11f0-a18d-1b26f69b8310 data_stream: dataset: zeek paths: - /nsm/zeek/logs/current/*.log + prospector.scanner.recursive_glob: true + prospector.scanner.exclude_files: + - >- + (broker|capture_loss|cluster|conn-summary|console|ecat_arp_info|known_certs|known_hosts|known_services|loaded_scripts|ntp|ocsp|packet_filter|reporter|stats|stderr|stdout).log$ + clean_inactive: -1 + parsers: null processors: - dissect: - tokenizer: '/nsm/zeek/logs/current/%{pipeline}.log' + tokenizer: /nsm/zeek/logs/current/%{pipeline}.log field: log.file.path trim_chars: .log target_prefix: '' @@ -427,18 +302,17 @@ inputs: regexp: pipeline: >- ^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm* - exclude_files: - - >- - broker|capture_loss|cluster|ecat_arp_info|known_hosts|known_services|loaded_scripts|ntp|ocsp|packet_filter|reporter|stats|stderr|stdout.log$ + file_identity.native: null + prospector.scanner.fingerprint.enabled: false - id: udp-udp-35051de0-46a5-11ee-8d5d-9f98c8182f60 name: syslog-udp-514 - revision: 3 + revision: 4 type: udp use_output: default meta: package: name: udp - version: 1.10.0 + version: data_stream: namespace: so package_policy_id: 35051de0-46a5-11ee-8d5d-9f98c8182f60 @@ -458,13 +332,13 @@ inputs: - syslog - id: tcp-tcp-33d37bb0-46a5-11ee-8d5d-9f98c8182f60 name: syslog-tcp-514 - revision: 3 + revision: 4 type: tcp use_output: default meta: package: name: tcp - version: 1.10.0 + version: data_stream: namespace: so package_policy_id: 33d37bb0-46a5-11ee-8d5d-9f98c8182f60 From 20382273089a7ac62245b6db1f86203279c79223 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 26 Jan 2026 14:01:58 -0600 Subject: [PATCH 07/10] remove reference to .fleet_final_pipeline-1 - configure global@custom ingest pipeline to run .fleet_final_pipeline-1 when available (heavynodes do not have this pipeline). - Update global@custom pipeline to remove error message related to sending EA logs through logstash (https://github.com/elastic/kibana/issues/183959) --- salt/elasticsearch/defaults.yaml | 1 - salt/elasticsearch/files/ingest/global@custom | 13 +++++++++++++ .../so-fleet_agent_id_verification-1.json | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 99403d9b8..67665a97b 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -691,7 +691,6 @@ elasticsearch: match_mapping_type: string settings: index: - final_pipeline: .fleet_final_pipeline-1 lifecycle: name: so-import-logs mapping: diff --git a/salt/elasticsearch/files/ingest/global@custom b/salt/elasticsearch/files/ingest/global@custom index 6bf36d1a3..bafb783a4 100644 --- a/salt/elasticsearch/files/ingest/global@custom +++ b/salt/elasticsearch/files/ingest/global@custom @@ -177,6 +177,19 @@ "description": "Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip" } }, + { + "pipeline": { + "name": ".fleet_final_pipeline-1", + "ignore_missing_pipeline": true + } + }, + { + "remove": { + "field": "event.agent_id_status", + "ignore_missing": true, + "if": "ctx?.event?.agent_id_status == 'auth_metadata_missing'" + } + }, { "remove": { "field": [ diff --git a/salt/elasticsearch/templates/component/elastic-agent/so-fleet_agent_id_verification-1.json b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_agent_id_verification-1.json index 99b3aa871..46e16bb44 100644 --- a/salt/elasticsearch/templates/component/elastic-agent/so-fleet_agent_id_verification-1.json +++ b/salt/elasticsearch/templates/component/elastic-agent/so-fleet_agent_id_verification-1.json @@ -2,7 +2,7 @@ "template": { "settings": { "index": { - "final_pipeline": ".fleet_final_pipeline-1" + "final_pipeline": "global@custom" } }, "mappings": { From ff4ec69f7ce6da3566e746e1a73f3144bac4a49e Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 27 Jan 2026 16:28:06 -0600 Subject: [PATCH 08/10] remove redis log collection on heavynodes (disabled) --- .../files/elastic-agent.yml.jinja | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/salt/elasticagent/files/elastic-agent.yml.jinja b/salt/elasticagent/files/elastic-agent.yml.jinja index ec4620efb..283bf9508 100644 --- a/salt/elasticagent/files/elastic-agent.yml.jinja +++ b/salt/elasticagent/files/elastic-agent.yml.jinja @@ -22,31 +22,6 @@ agent: metrics: false features: {} inputs: - - id: logfile-redis-8b7c8390-25ef-11f0-a18d-1b26f69b8310 - name: redis-logs - revision: 3 - type: logfile - use_output: default - meta: - package: - name: redis - version: - data_stream: - namespace: default - package_policy_id: 8b7c8390-25ef-11f0-a18d-1b26f69b8310 - streams: - - id: logfile-redis.log-8b7c8390-25ef-11f0-a18d-1b26f69b8310 - data_stream: - dataset: redis.log - type: logs - paths: - - /opt/so/log/redis/redis-server.log - tags: - - redis-log - exclude_files: - - .gz$ - exclude_lines: - - ^\s+[\-`('.|_] - id: filestream-filestream-85820eb0-25ef-11f0-a18d-1b26f69b8310 name: import-suricata-logs revision: 3 From e5226b50edb36e2cae51a77bc5523cf93193931e Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 27 Jan 2026 16:37:23 -0600 Subject: [PATCH 09/10] disable logstash metrics collection on nodes not running logstash + fleet nodes --- salt/telegraf/etc/telegraf.conf | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index d2cb87057..e74c79f6c 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -7,6 +7,7 @@ {%- set UNIQUEID = salt['pillar.get']('sensor:uniqueid', '') %} {%- set ZEEK_ENABLED = salt['pillar.get']('zeek:enabled', True) %} {%- set MDENGINE = GLOBALS.md_engine %} +{%- set LOGSTASH_ENABLED = salt['pillar.get']('logstash:enabled', False) %} # Global tags can be specified here in key="value" format. [global_tags] role = "{{ GLOBALS.role.split('-') | last }}" @@ -241,12 +242,8 @@ # ## Use TLS but skip chain & host verification # # insecure_skip_verify = false -{%- set logstash_metrics_roles = ['so-searchnode','so-standalone','so-managersearch','so-heavynode'] %} -{%- if GLOBALS.pipeline != "KAFKA" %} -{%- set logstash_metrics_roles = logstash_metrics_roles + ['so-manager', 'so-receiver'] %} -{%- endif %} - -{%- if grains.role in logstash_metrics_roles %} +{#- Fleet nodes do not have pillar access to logstash credentials #} +{%- if LOGSTASH_ENABLED and grains.role != 'so-fleet' %} [[inputs.logstash]] url = "http://localhost:9600" collect = ["pipelines"] From 057131dce7226089fe8783bf55e6b3b7e77570bb Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 27 Jan 2026 16:39:07 -0600 Subject: [PATCH 10/10] disable redis on heavynodes -- no longer in use --- salt/manager/tools/sbin/so-minion | 1 - salt/manager/tools/sbin/soup | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 045c05c42..417b1eaf3 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -839,7 +839,6 @@ function createHEAVYNODE() { add_elastic_agent_to_minion || return 1 add_sensor_to_minion || return 1 add_strelka_to_minion || return 1 - add_redis_to_minion || return 1 add_telegraf_to_minion || return 1 } diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index e278b4361..92f3b6d14 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -353,6 +353,22 @@ disable_logstash_heavynodes() { done } +disable_redis_heavynodes() { + local c=0 + printf "\nChecking for heavynodes and disabling Redis if they exist\n" + for file in /opt/so/saltstack/local/pillar/minions/*.sls; do + if [[ "$file" =~ "_heavynode.sls" && ! "$file" =~ "/opt/so/saltstack/local/pillar/minions/adv_" ]]; then + c=1 + echo "Disabling Redis for: $file" + so-yaml.py replace "$file" redis.enabled False + fi + done + + if [[ "$c" != 0 ]]; then + FINAL_MESSAGE_QUEUE+=("Redis has been disabled on all heavynodes.") + fi +} + enable_highstate() { echo "Enabling highstate." salt-call state.enable highstate -l info --local @@ -674,6 +690,8 @@ post_to_2.4.210() { rollover_index "logs-kratos-so" + disable_redis_heavynodes + echo "Regenerating Elastic Agent Installers" /sbin/so-elastic-agent-gen-installers