From 8101bc4941c230418aeb3b8a675739392b3a0139 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 6 Apr 2026 15:08:30 -0500 Subject: [PATCH 01/24] ES 9.3.2 --- .../elastic-defend-endpoints.json | 2 +- .../grid-nodes_general/import-evtx-logs.json | 2 +- salt/elasticsearch/defaults.yaml | 2 +- ...nse.log-1.23.1 => logs-pfsense.log-1.25.1} | 83 ++++++++++++++++--- ...icata => logs-pfsense.log-1.25.1-suricata} | 0 .../tools/sbin_jinja/so-kibana-space-defaults | 2 +- 6 files changed, 76 insertions(+), 15 deletions(-) rename salt/elasticsearch/files/ingest/{logs-pfsense.log-1.23.1 => logs-pfsense.log-1.25.1} (74%) rename salt/elasticsearch/files/ingest/{logs-pfsense.log-1.23.1-suricata => logs-pfsense.log-1.25.1-suricata} (100%) diff --git a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json index debfc73a3..c27da26f7 100644 --- a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json +++ b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json @@ -5,7 +5,7 @@ "package": { "name": "endpoint", "title": "Elastic Defend", - "version": "9.0.2", + "version": "9.3.0", "requires_root": true }, "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json index 50ffd5dc7..3066303d9 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json @@ -23,7 +23,7 @@ "\\.gz$" ], "include_files": [], - "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.6.1\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.1.2\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.6.1\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.6.1\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.1.2\n- add_fields:\n target: data_stream\n fields:\n dataset: import", + "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.13.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.6.0\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.13.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.13.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.6.0\n- add_fields:\n target: data_stream\n fields:\n dataset: import", "tags": [ "import" ], diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index d0ab0f959..f355601dc 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1,6 +1,6 @@ elasticsearch: enabled: false - version: 9.0.8 + version: 9.3.2 index_clean: true vm: max_map_count: 1048576 diff --git a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.1 b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.1 similarity index 74% rename from salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.1 rename to salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.1 index d3354f363..3037ce77a 100644 --- a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.1 +++ b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.1 @@ -10,24 +10,28 @@ "processors": [ { "set": { + "tag": "set_ecs_version_f5923549", "field": "ecs.version", "value": "8.17.0" } }, { "set": { + "tag": "set_observer_vendor_ad9d35cc", "field": "observer.vendor", "value": "netgate" } }, { "set": { + "tag": "set_observer_type_5dddf3ba", "field": "observer.type", "value": "firewall" } }, { "rename": { + "tag": "rename_message_to_event_original_56a77271", "field": "message", "target_field": "event.original", "ignore_missing": true, @@ -36,12 +40,14 @@ }, { "set": { + "tag": "set_event_kind_de80643c", "field": "event.kind", "value": "event" } }, { "set": { + "tag": "set_event_timezone_4ca44cac", "field": "event.timezone", "value": "{{{_tmp.tz_offset}}}", "if": "ctx._tmp?.tz_offset != null && ctx._tmp?.tz_offset != 'local'" @@ -49,6 +55,7 @@ }, { "grok": { + "tag": "grok_event_original_27d9c8c7", "description": "Parse syslog header", "field": "event.original", "patterns": [ @@ -72,6 +79,7 @@ }, { "date": { + "tag": "date__tmp_timestamp8601_to_timestamp_6ac9d3ce", "if": "ctx._tmp.timestamp8601 != null", "field": "_tmp.timestamp8601", "target_field": "@timestamp", @@ -82,6 +90,7 @@ }, { "date": { + "tag": "date__tmp_timestamp_to_timestamp_f21e536e", "if": "ctx.event?.timezone != null && ctx._tmp?.timestamp != null", "field": "_tmp.timestamp", "target_field": "@timestamp", @@ -95,6 +104,7 @@ }, { "grok": { + "tag": "grok_process_name_cef3d489", "description": "Set Event Provider", "field": "process.name", "patterns": [ @@ -107,71 +117,83 @@ }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-firewall", + "tag": "pipeline_e16851a7", + "name": "logs-pfsense.log-1.25.1-firewall", "if": "ctx.event.provider == 'filterlog'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-openvpn", + "tag": "pipeline_828590b5", + "name": "logs-pfsense.log-1.25.1-openvpn", "if": "ctx.event.provider == 'openvpn'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-ipsec", + "tag": "pipeline_9d37039c", + "name": "logs-pfsense.log-1.25.1-ipsec", "if": "ctx.event.provider == 'charon'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-dhcp", + "tag": "pipeline_ad56bbca", + "name": "logs-pfsense.log-1.25.1-dhcp", "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\"].contains(ctx.event.provider)" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-unbound", + "tag": "pipeline_dd85553d", + "name": "logs-pfsense.log-1.25.1-unbound", "if": "ctx.event.provider == 'unbound'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-haproxy", + "tag": "pipeline_720ed255", + "name": "logs-pfsense.log-1.25.1-haproxy", "if": "ctx.event.provider == 'haproxy'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-php-fpm", + "tag": "pipeline_456beba5", + "name": "logs-pfsense.log-1.25.1-php-fpm", "if": "ctx.event.provider == 'php-fpm'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-squid", + "tag": "pipeline_a0d89375", + "name": "logs-pfsense.log-1.25.1-squid", "if": "ctx.event.provider == 'squid'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-snort", + "tag": "pipeline_c2f1ed55", + "name": "logs-pfsense.log-1.25.1-snort", "if": "ctx.event.provider == 'snort'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-suricata", + "tag":"pipeline_33db1c9e", + "name": "logs-pfsense.log-1.25.1-suricata", "if": "ctx.event.provider == 'suricata'" } }, { "drop": { + "tag": "drop_9d7c46f8", "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"snort\", \"suricata\"].contains(ctx.event?.provider)" } }, { "append": { + "tag": "append_event_category_4780a983", "field": "event.category", "value": "network", "if": "ctx.network != null" @@ -179,6 +201,7 @@ }, { "convert": { + "tag": "convert_source_address_to_source_ip_f5632a20", "field": "source.address", "target_field": "source.ip", "type": "ip", @@ -188,6 +211,7 @@ }, { "convert": { + "tag": "convert_destination_address_to_destination_ip_f1388f0c", "field": "destination.address", "target_field": "destination.ip", "type": "ip", @@ -197,6 +221,7 @@ }, { "set": { + "tag": "set_network_type_1f1d940a", "field": "network.type", "value": "ipv6", "if": "ctx.source?.ip != null && ctx.source.ip.contains(\":\")" @@ -204,6 +229,7 @@ }, { "set": { + "tag": "set_network_type_69deca38", "field": "network.type", "value": "ipv4", "if": "ctx.source?.ip != null && ctx.source.ip.contains(\".\")" @@ -211,6 +237,7 @@ }, { "geoip": { + "tag": "geoip_source_ip_to_source_geo_da2e41b2", "field": "source.ip", "target_field": "source.geo", "ignore_missing": true @@ -218,6 +245,7 @@ }, { "geoip": { + "tag": "geoip_destination_ip_to_destination_geo_ab5e2968", "field": "destination.ip", "target_field": "destination.geo", "ignore_missing": true @@ -225,6 +253,7 @@ }, { "geoip": { + "tag": "geoip_source_ip_to_source_as_28d69883", "ignore_missing": true, "database_file": "GeoLite2-ASN.mmdb", "field": "source.ip", @@ -237,6 +266,7 @@ }, { "geoip": { + "tag": "geoip_destination_ip_to_destination_as_8a007787", "database_file": "GeoLite2-ASN.mmdb", "field": "destination.ip", "target_field": "destination.as", @@ -249,6 +279,7 @@ }, { "rename": { + "tag": "rename_source_as_asn_to_source_as_number_a917047d", "field": "source.as.asn", "target_field": "source.as.number", "ignore_missing": true @@ -256,6 +287,7 @@ }, { "rename": { + "tag": "rename_source_as_organization_name_to_source_as_organization_name_f1362d0b", "field": "source.as.organization_name", "target_field": "source.as.organization.name", "ignore_missing": true @@ -263,6 +295,7 @@ }, { "rename": { + "tag": "rename_destination_as_asn_to_destination_as_number_3b459fcd", "field": "destination.as.asn", "target_field": "destination.as.number", "ignore_missing": true @@ -270,6 +303,7 @@ }, { "rename": { + "tag": "rename_destination_as_organization_name_to_destination_as_organization_name_814bd459", "field": "destination.as.organization_name", "target_field": "destination.as.organization.name", "ignore_missing": true @@ -277,12 +311,14 @@ }, { "community_id": { + "tag": "community_id_d2308e7a", "target_field": "network.community_id", "ignore_failure": true } }, { "grok": { + "tag": "grok_observer_ingress_interface_name_968018d3", "field": "observer.ingress.interface.name", "patterns": [ "%{DATA}.%{NONNEGINT:observer.ingress.vlan.id}" @@ -293,6 +329,7 @@ }, { "set": { + "tag": "set_network_vlan_id_efd4d96a", "field": "network.vlan.id", "copy_from": "observer.ingress.vlan.id", "ignore_empty_value": true @@ -300,6 +337,7 @@ }, { "append": { + "tag": "append_related_ip_c1a6356b", "field": "related.ip", "value": "{{{destination.ip}}}", "allow_duplicates": false, @@ -308,6 +346,7 @@ }, { "append": { + "tag": "append_related_ip_8121c591", "field": "related.ip", "value": "{{{source.ip}}}", "allow_duplicates": false, @@ -316,6 +355,7 @@ }, { "append": { + "tag": "append_related_ip_53b62ed8", "field": "related.ip", "value": "{{{source.nat.ip}}}", "allow_duplicates": false, @@ -324,6 +364,7 @@ }, { "append": { + "tag": "append_related_hosts_6f162628", "field": "related.hosts", "value": "{{{destination.domain}}}", "if": "ctx.destination?.domain != null" @@ -331,6 +372,7 @@ }, { "append": { + "tag": "append_related_user_c036eec2", "field": "related.user", "value": "{{{user.name}}}", "if": "ctx.user?.name != null" @@ -338,6 +380,7 @@ }, { "set": { + "tag": "set_network_direction_cb1e3125", "field": "network.direction", "value": "{{{network.direction}}}bound", "if": "ctx.network?.direction != null && ctx.network?.direction =~ /^(in|out)$/" @@ -345,6 +388,7 @@ }, { "remove": { + "tag": "remove_a82e20f2", "field": [ "_tmp" ], @@ -353,11 +397,21 @@ }, { "script": { + "tag": "script_a7f2c062", "lang": "painless", "description": "This script processor iterates over the whole document to remove fields with null values.", "source": "void handleMap(Map map) {\n for (def x : map.values()) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n map.values().removeIf(v -> v == null || (v instanceof String && v == \"-\"));\n}\nvoid handleList(List list) {\n for (def x : list) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n}\nhandleMap(ctx);\n" } }, + { + "append": { + "tag": "append_preserve_original_event_on_error", + "field": "tags", + "value": "preserve_original_event", + "allow_duplicates": false, + "if": "ctx.error?.message != null" + } + }, { "pipeline": { "name": "global@custom", @@ -405,7 +459,14 @@ { "append": { "field": "error.message", - "value": "{{{ _ingest.on_failure_message }}}" + "value": "Processor '{{{ _ingest.on_failure_processor_type }}}' {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}'" + } + }, + { + "append": { + "field": "tags", + "value": "preserve_original_event", + "allow_duplicates": false } } ] diff --git a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.1-suricata b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.1-suricata similarity index 100% rename from salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.1-suricata rename to salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.1-suricata diff --git a/salt/kibana/tools/sbin_jinja/so-kibana-space-defaults b/salt/kibana/tools/sbin_jinja/so-kibana-space-defaults index fcb80e606..d0447f514 100755 --- a/salt/kibana/tools/sbin_jinja/so-kibana-space-defaults +++ b/salt/kibana/tools/sbin_jinja/so-kibana-space-defaults @@ -9,5 +9,5 @@ SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http: # Disable certain Features from showing up in the Kibana UI echo echo "Setting up default Kibana Space:" -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","entityManager"]} ' >> /opt/so/log/kibana/misc.log +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV1","entityManager","streams","cloudConnect","slo"]} ' >> /opt/so/log/kibana/misc.log echo From 51a3c04c3d945cd1ffb02bf38017ca194a62c6cd Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 6 Apr 2026 17:35:08 -0500 Subject: [PATCH 02/24] foxtrot version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index fd2a01863..0e0d1ae9a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.0 +3.0.0-foxtrot \ No newline at end of file From dd56e7f1aca5b4528c5050e090b1f73599d83ae7 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 7 Apr 2026 11:08:10 -0500 Subject: [PATCH 03/24] filestream integration policy updates --- .../elastic-agent-monitor.json | 17 ++++++++++------- .../grid-nodes_general/hydra-logs.json | 14 ++++++++++---- .../grid-nodes_general/idh-logs.json | 14 ++++++++++---- .../grid-nodes_general/import-evtx-logs.json | 14 ++++++++++---- .../import-suricata-logs.json | 14 ++++++++++---- .../grid-nodes_general/rita-logs.json | 14 ++++++++++---- .../grid-nodes_general/so-ip-mappings.json | 14 ++++++++++---- .../grid-nodes_general/soc-auth-sync-logs.json | 14 ++++++++++---- .../grid-nodes_general/soc-detections-logs.json | 14 ++++++++++---- .../grid-nodes_general/soc-salt-relay-logs.json | 14 ++++++++++---- .../grid-nodes_general/soc-sensoroni-logs.json | 14 ++++++++++---- .../grid-nodes_general/soc-server-logs.json | 14 ++++++++++---- .../grid-nodes_general/strelka-logs.json | 14 ++++++++++---- .../grid-nodes_general/suricata-logs.json | 14 ++++++++++---- 14 files changed, 140 insertions(+), 59 deletions(-) diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/elastic-agent-monitor.json b/salt/elasticfleet/files/integrations/grid-nodes_general/elastic-agent-monitor.json index 0be40a3d3..3eec63d26 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/elastic-agent-monitor.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/elastic-agent-monitor.json @@ -6,21 +6,23 @@ "name": "agent-monitor", "namespace": "", "description": "", + "policy_id": "so-grid-nodes_general", "policy_ids": [ "so-grid-nodes_general" ], - "output_id": null, "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/opt/so/log/agents/agent-monitor.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "agentmonitor", "pipeline": "elasticagent.monitor", "parsers": "", @@ -34,15 +36,16 @@ "ignore_older": "72h", "clean_inactive": -1, "harvester_limit": 0, - "fingerprint": true, + "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": 64, - "file_identity_native": false, + "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } } - } + }, + "force": true } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json index a4f944ba5..5dcd3012d 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "hydra-logs", + "namespace": "so", "description": "Hydra logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/opt/so/log/hydra/hydra.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "hydra", "pipeline": "hydra", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -34,10 +40,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json index fef9c57fb..afaf77f0c 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "idh-logs", + "namespace": "so", "description": "IDH integration", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/nsm/idh/opencanary.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "idh", "pipeline": "common", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -31,10 +37,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json index 3066303d9..0e42a0dfb 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "import-evtx-logs", + "namespace": "so", "description": "Import Windows EVTX logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/nsm/import/*/evtx/*.json" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "import", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", "exclude_files": [ @@ -33,10 +39,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json index b8f3b0b29..3148b38e8 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "import-suricata-logs", + "namespace": "so", "description": "Import Suricata logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/nsm/import/*/suricata/eve*.json" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "import", "pipeline": "suricata.common", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -32,10 +38,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json index 70259c3cf..f807c3b70 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json @@ -4,14 +4,18 @@ "version": "" }, "name": "rita-logs", + "namespace": "so", "description": "RITA Logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ @@ -19,6 +23,8 @@ "/nsm/rita/exploded-dns.csv", "/nsm/rita/long-connections.csv" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "rita", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", "exclude_files": [ @@ -33,10 +39,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json b/salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json index a14e63559..24ed188f2 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json @@ -4,19 +4,25 @@ "version": "" }, "name": "so-ip-mappings", + "namespace": "so", "description": "IP Description mappings", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/nsm/custom-mappings/ip-descriptions.csv" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "hostnamemappings", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", "exclude_files": [ @@ -32,10 +38,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json index f4fd38e9d..c04b738d3 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "soc-auth-sync-logs", + "namespace": "so", "description": "Security Onion - Elastic Auth Sync - Logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/opt/so/log/soc/sync.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "soc", "pipeline": "common", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -31,10 +37,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json index f1bdbc922..9d7812e42 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json @@ -4,20 +4,26 @@ "version": "" }, "name": "soc-detections-logs", + "namespace": "so", "description": "Security Onion Console - Detections Logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/opt/so/log/soc/detections_runtime-status_sigma.log", "/opt/so/log/soc/detections_runtime-status_yara.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "soc", "pipeline": "common", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -35,10 +41,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json index cb08d5b12..d1fa8b630 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "soc-salt-relay-logs", + "namespace": "so", "description": "Security Onion - Salt Relay - Logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/opt/so/log/soc/salt-relay.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "soc", "pipeline": "common", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -33,10 +39,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json index 11e686c3d..467544c9d 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "soc-sensoroni-logs", + "namespace": "so", "description": "Security Onion - Sensoroni - Logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/opt/so/log/sensoroni/sensoroni.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "soc", "pipeline": "common", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -31,10 +37,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json index decb6b22a..37eb02ab1 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "soc-server-logs", + "namespace": "so", "description": "Security Onion Console Logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/opt/so/log/soc/sensoroni-server.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "soc", "pipeline": "common", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -33,10 +39,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json index 1f0203a91..3091baf44 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "strelka-logs", + "namespace": "so", "description": "Strelka Logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/nsm/strelka/log/strelka.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "strelka", "pipeline": "strelka.file", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -31,10 +37,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json index 26dae5225..bb5cfd2c3 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json @@ -4,19 +4,25 @@ "version": "" }, "name": "suricata-logs", + "namespace": "so", "description": "Suricata integration", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/nsm/suricata/eve*.json" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "suricata", "pipeline": "suricata.common", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -31,10 +37,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } From 2166bb749af57ee41702dd477d5b66acdf85247d Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 8 Apr 2026 14:59:05 -0400 Subject: [PATCH 04/24] ensure max-files is 1 at minimum --- salt/suricata/map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/suricata/map.jinja b/salt/suricata/map.jinja index 944e0e34d..f7059b293 100644 --- a/salt/suricata/map.jinja +++ b/salt/suricata/map.jinja @@ -33,7 +33,7 @@ {% do SURICATAMERGED.config.outputs['pcap-log'].update({'conditional': SURICATAMERGED.pcap.conditional}) %} {% do SURICATAMERGED.config.outputs['pcap-log'].update({'dir': SURICATAMERGED.pcap.dir}) %} {# multiply maxsize by 1000 since it is saved in GB, i.e. 52 = 52000MB. filesize is also saved in MB and we strip the MB and convert to int #} -{% set maxfiles = (SURICATAMERGED.pcap.maxsize * 1000 / (SURICATAMERGED.pcap.filesize[:-2] | int) / SURICATAMERGED.config['af-packet'].threads | int) | round | int %} +{% set maxfiles = ([1, (SURICATAMERGED.pcap.maxsize * 1000 / (SURICATAMERGED.pcap.filesize[:-2] | int) / SURICATAMERGED.config['af-packet'].threads | int) | round(0, 'ceil') | int] | max) %} {% do SURICATAMERGED.config.outputs['pcap-log'].update({'max-files': maxfiles}) %} {% endif %} From 28d31f48409f0a358456cfdbc14d67325d54672d Mon Sep 17 00:00:00 2001 From: Matthew Wright Date: Wed, 8 Apr 2026 15:25:51 -0400 Subject: [PATCH 05/24] add charsPerTokenEstimate --- salt/soc/defaults.yaml | 1 + salt/soc/soc_soc.yaml | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 0bde8f20e..cc80758fc 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -2687,4 +2687,5 @@ soc: lowBalanceColorAlert: 500000 enabled: true adapter: SOAI + charsPerTokenEstimate: 4 diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index c5f96894d..d4e908637 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -761,7 +761,7 @@ soc: required: True - field: origin label: Country of Origin for the Model Training - required: false + required: False - field: contextLimitSmall label: Context Limit (Small) forcedType: int @@ -779,6 +779,10 @@ soc: - field: enabled label: Enabled forcedType: bool + - field: charsPerTokenEstimate + label: Characters per Token Estimate + forcedType: float + required: False apiTimeoutMs: description: Duration (in milliseconds) to wait for a response from the SOC server API before giving up and showing an error on the SOC UI. global: True From 9ec4a26f97dd68fb02ddd6d422270fdf9700ec25 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Thu, 9 Apr 2026 10:18:36 -0400 Subject: [PATCH 06/24] define options in annotation files --- salt/global/soc_global.yaml | 6 ------ salt/influxdb/soc_influxdb.yaml | 13 ++++++++++--- salt/kafka/soc_kafka.yaml | 18 ++++++++++++++---- salt/kratos/soc_kratos.yaml | 13 +++++++++---- salt/suricata/soc_suricata.yaml | 10 +++++++--- 5 files changed, 40 insertions(+), 20 deletions(-) diff --git a/salt/global/soc_global.yaml b/salt/global/soc_global.yaml index 33abbf690..c15f3eb98 100644 --- a/salt/global/soc_global.yaml +++ b/salt/global/soc_global.yaml @@ -11,18 +11,14 @@ global: regexFailureMessage: You must enter a valid IP address or CIDR. mdengine: description: Which engine to use for meta data generation. Options are ZEEK and SURICATA. - regex: ^(ZEEK|SURICATA)$ options: - ZEEK - SURICATA - regexFailureMessage: You must enter either ZEEK or SURICATA. global: True pcapengine: description: Which engine to use for generating pcap. Currently only SURICATA is supported. - regex: ^(SURICATA)$ options: - SURICATA - regexFailureMessage: You must enter either SURICATA. global: True ids: description: Which IDS engine to use. Currently only Suricata is supported. @@ -42,11 +38,9 @@ global: advanced: True pipeline: description: Sets which pipeline technology for events to use. The use of Kafka requires a Security Onion Pro license. - regex: ^(REDIS|KAFKA)$ options: - REDIS - KAFKA - regexFailureMessage: You must enter either REDIS or KAFKA. global: True advanced: True repo_host: diff --git a/salt/influxdb/soc_influxdb.yaml b/salt/influxdb/soc_influxdb.yaml index 3dbf0875b..2b6bffe49 100644 --- a/salt/influxdb/soc_influxdb.yaml +++ b/salt/influxdb/soc_influxdb.yaml @@ -85,7 +85,10 @@ influxdb: description: The log level to use for outputting log statements. Allowed values are debug, info, or error. global: True advanced: false - regex: ^(info|debug|error)$ + options: + - info + - debug + - error helpLink: influxdb metrics-disabled: description: If true, the HTTP endpoint that exposes internal InfluxDB metrics will be inaccessible. @@ -140,7 +143,9 @@ influxdb: description: Determines the type of storage used for secrets. Allowed values are bolt or vault. global: True advanced: True - regex: ^(bolt|vault)$ + options: + - bolt + - vault helpLink: influxdb session-length: description: Number of minutes that a user login session can remain authenticated. @@ -260,7 +265,9 @@ influxdb: description: The type of data store to use for HTTP resources. Allowed values are disk or memory. Memory should not be used for production Security Onion installations. global: True advanced: True - regex: ^(disk|memory)$ + options: + - disk + - memory helpLink: influxdb tls-cert: description: The container path to the certificate to use for TLS encryption of the HTTP requests and responses. diff --git a/salt/kafka/soc_kafka.yaml b/salt/kafka/soc_kafka.yaml index b8d0c7c32..85469b8a4 100644 --- a/salt/kafka/soc_kafka.yaml +++ b/salt/kafka/soc_kafka.yaml @@ -128,10 +128,13 @@ kafka: title: ssl.keystore.password sensitive: True helpLink: kafka - ssl_x_keystore_x_type: + ssl_x_keystore_x_type: description: The key store file format. title: ssl.keystore.type - regex: ^(JKS|PKCS12|PEM)$ + options: + - JKS + - PKCS12 + - PEM helpLink: kafka ssl_x_truststore_x_location: description: The trust store file location within the Docker container. @@ -160,7 +163,11 @@ kafka: security_x_protocol: description: 'Broker communication protocol. Options are: SASL_SSL, PLAINTEXT, SSL, SASL_PLAINTEXT' title: security.protocol - regex: ^(SASL_SSL|PLAINTEXT|SSL|SASL_PLAINTEXT) + options: + - SASL_SSL + - PLAINTEXT + - SSL + - SASL_PLAINTEXT helpLink: kafka ssl_x_keystore_x_location: description: The key store file location within the Docker container. @@ -174,7 +181,10 @@ kafka: ssl_x_keystore_x_type: description: The key store file format. title: ssl.keystore.type - regex: ^(JKS|PKCS12|PEM)$ + options: + - JKS + - PKCS12 + - PEM helpLink: kafka ssl_x_truststore_x_location: description: The trust store file location within the Docker container. diff --git a/salt/kratos/soc_kratos.yaml b/salt/kratos/soc_kratos.yaml index 1cd2728c8..07359bcab 100644 --- a/salt/kratos/soc_kratos.yaml +++ b/salt/kratos/soc_kratos.yaml @@ -21,8 +21,12 @@ kratos: description: "Specify the provider type. Required. Valid values are: auth0, generic, github, google, microsoft" global: True forcedType: string - regex: "auth0|generic|github|google|microsoft" - regexFailureMessage: "Valid values are: auth0, generic, github, google, microsoft" + options: + - auth0 + - generic + - github + - google + - microsoft helpLink: oidc client_id: description: Specify the client ID, also referenced as the application ID. Required. @@ -43,8 +47,9 @@ kratos: description: The source of the subject identifier. Typically 'userinfo'. Only used when provider is 'microsoft'. global: True forcedType: string - regex: me|userinfo - regexFailureMessage: "Valid values are: me, userinfo" + options: + - me + - userinfo helpLink: oidc auth_url: description: Provider's auth URL. Required when provider is 'generic'. diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index c85b876a9..ce6b7d008 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -64,8 +64,10 @@ suricata: helpLink: suricata conditional: description: Set to "all" to record PCAP for all flows. Set to "alerts" to only record PCAP for Suricata alerts. Set to "tag" to only record PCAP for tagged rules. - regex: ^(all|alerts|tag)$ - regexFailureMessage: You must enter either all, alert or tag. + options: + - all + - alerts + - tag helpLink: suricata dir: description: Parent directory to store PCAP. @@ -83,7 +85,9 @@ suricata: advanced: True cluster-type: advanced: True - regex: ^(cluster_flow|cluster_qm)$ + options: + - cluster_flow + - cluster_qm defrag: description: Enable defragmentation of IP packets before processing. forcedType: bool From f0b67a415accd492e1e75b233c0dfe7a9f8c2fbe Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 9 Apr 2026 12:40:55 -0500 Subject: [PATCH 07/24] more filestream integration policy updates --- .../grid-nodes_general/import-zeek-logs.json | 11 +++++++++-- .../grid-nodes_general/kratos-logs.json | 14 ++++++++++---- .../grid-nodes_general/zeek-logs.json | 12 +++++++++--- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json index ac03f3c1d..c1fd7f147 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json @@ -9,16 +9,22 @@ "namespace": "so", "description": "Zeek Import logs", "policy_id": "so-grid-nodes_general", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/nsm/import/*/zeek/logs/*.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "import", "pipeline": "", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -34,7 +40,8 @@ "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json index 545588521..83d153439 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json @@ -15,19 +15,25 @@ "version": "" }, "name": "kratos-logs", + "namespace": "so", "description": "Kratos logs", "policy_id": "so-grid-nodes_general", - "namespace": "so", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kratos/kratos.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "kratos", "pipeline": "kratos", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", @@ -48,10 +54,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json index 4af2b2921..9797b9e75 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json @@ -9,16 +9,22 @@ "namespace": "so", "description": "Zeek logs", "policy_id": "so-grid-nodes_general", + "policy_ids": [ + "so-grid-nodes_general" + ], + "vars": {}, "inputs": { "filestream-filestream": { "enabled": true, "streams": { - "filestream.generic": { + "filestream.filestream": { "enabled": true, "vars": { "paths": [ "/nsm/zeek/logs/current/*.log" ], + "compression_gzip": false, + "use_logs_stream": false, "data_stream.dataset": "zeek", "parsers": "#- ndjson:\n# target: \"\"\n# message_key: msg\n#- multiline:\n# type: count\n# count_lines: 3\n", "exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%})(\\..+)?\\.log$"], @@ -30,10 +36,10 @@ "harvester_limit": 0, "fingerprint": false, "fingerprint_offset": 0, - "fingerprint_length": "64", "file_identity_native": true, "exclude_lines": [], - "include_lines": [] + "include_lines": [], + "delete_enabled": false } } } From 89e49d0bf35dcff8202c27462b3c966601737af7 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 9 Apr 2026 16:44:51 -0500 Subject: [PATCH 08/24] rework elasticsearch index template generation --- salt/elasticfleet/content-defaults.map.jinja | 123 ++++++++++++++++++ salt/elasticfleet/input-defaults.map.jinja | 123 ++++++++++++++++++ .../integration-defaults.map.jinja | 27 +--- .../tools/sbin/so-elastic-fleet-common | 28 +++- ...o-elastic-fleet-optional-integrations-load | 15 ++- salt/elasticsearch/config.sls | 7 + salt/elasticsearch/enabled.sls | 58 +++++---- salt/elasticsearch/template.map.jinja | 80 +++++++++--- 8 files changed, 392 insertions(+), 69 deletions(-) create mode 100644 salt/elasticfleet/content-defaults.map.jinja create mode 100644 salt/elasticfleet/input-defaults.map.jinja diff --git a/salt/elasticfleet/content-defaults.map.jinja b/salt/elasticfleet/content-defaults.map.jinja new file mode 100644 index 000000000..f4237d6d1 --- /dev/null +++ b/salt/elasticfleet/content-defaults.map.jinja @@ -0,0 +1,123 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use + this file except in compliance with the Elastic License 2.0. #} + + +{% import_json '/opt/so/state/esfleet_content_package_components.json' as ADDON_CONTENT_PACKAGE_COMPONENTS %} +{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %} +{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} + +{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %} +{% set ADDON_CONTENT_INTEGRATION_DEFAULTS = {} %} +{% set DEBUG_STUFF = {} %} + +{% for pkg in ADDON_CONTENT_PACKAGE_COMPONENTS %} +{% if pkg.name in CORE_ESFLEET_PACKAGES %} +{# skip core content packages #} +{% elif pkg.name not in CORE_ESFLEET_PACKAGES %} +{# generate defaults for each content package #} +{% if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0%} +{% for pattern in pkg.dataStreams %} +{# in ES 9.3.2 'input' type integrations no longer create default component templates and instead they wait for user input during 'integration' setup (fleet ui config) + title: generic is an artifact of that and is not in use #} +{% if pattern.title == "generic" %} +{% continue %} +{% endif %} +{% if "metrics-" in pattern.name %} +{% set integration_type = "metrics-" %} +{% elif "logs-" in pattern.name %} +{% set integration_type = "logs-" %} +{% else %} +{% set integration_type = "" %} +{% endif %} +{# on content integrations the component name is user defined at the time it is added to an agent policy #} +{% set component_name = pattern.title %} +{% set index_pattern = pattern.name %} +{# component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #} +{% set component_name_x = component_name.replace(".","_x_") %} +{# pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #} +{% set integration_key = "so-" ~ integration_type ~ pkg.name + '_x_' ~ component_name_x %} +{# Default integration settings #} +{% set integration_defaults = { + "index_sorting": false, + "index_template": { + "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"], + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"], + "index_patterns": [index_pattern], + "priority": 501, + "template": { + "settings": { + "index": { + "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"}, + "number_of_replicas": 0 + } + } + } + }, + "policy": { + "phases": { + "cold": { + "actions": { + "allocate":{ + "number_of_replicas": "" + }, + "set_priority": {"priority": 0} + }, + "min_age": "60d" + }, + "delete": { + "actions": { + "delete": {} + }, + "min_age": "365d" + }, + "hot": { + "actions": { + "rollover": { + "max_age": "30d", + "max_primary_shard_size": "50gb" + }, + "forcemerge":{ + "max_num_segments": "" + }, + "shrink":{ + "max_primary_shard_size": "", + "method": "COUNT", + "number_of_shards": "" + }, + "set_priority": {"priority": 100} + }, + "min_age": "0ms" + }, + "warm": { + "actions": { + "allocate": { + "number_of_replicas": "" + }, + "forcemerge": { + "max_num_segments": "" + }, + "shrink":{ + "max_primary_shard_size": "", + "method": "COUNT", + "number_of_shards": "" + }, + "set_priority": {"priority": 50} + }, + "min_age": "30d" + } + } + } + } %} + + +{% do ADDON_CONTENT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %} +{% endfor %} +{% else %} +{% endif %} +{% endif %} +{% endfor %} diff --git a/salt/elasticfleet/input-defaults.map.jinja b/salt/elasticfleet/input-defaults.map.jinja new file mode 100644 index 000000000..a02844330 --- /dev/null +++ b/salt/elasticfleet/input-defaults.map.jinja @@ -0,0 +1,123 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use + this file except in compliance with the Elastic License 2.0. #} + + +{% import_json '/opt/so/state/esfleet_input_package_components.json' as ADDON_INPUT_PACKAGE_COMPONENTS %} +{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %} +{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} + +{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %} +{% set ADDON_INPUT_INTEGRATION_DEFAULTS = {} %} +{% set DEBUG_STUFF = {} %} + +{% for pkg in ADDON_INPUT_PACKAGE_COMPONENTS %} +{% if pkg.name in CORE_ESFLEET_PACKAGES %} +{# skip core input packages #} +{% elif pkg.name not in CORE_ESFLEET_PACKAGES %} +{# generate defaults for each input package #} +{% if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0 %} +{% for pattern in pkg.dataStreams %} +{# in ES 9.3.2 'input' type integrations no longer create default component templates and instead they wait for user input during 'integration' setup (fleet ui config) + title: generic is an artifact of that and is not in use #} +{% if pattern.title == "generic" %} +{% continue %} +{% endif %} +{% if "metrics-" in pattern.name %} +{% set integration_type = "metrics-" %} +{% elif "logs-" in pattern.name %} +{% set integration_type = "logs-" %} +{% else %} +{% set integration_type = "" %} +{% endif %} +{# on input integrations the component name is user defined at the time it is added to an agent policy #} +{% set component_name = pattern.title %} +{% set index_pattern = pattern.name %} +{# component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #} +{% set component_name_x = component_name.replace(".","_x_") %} +{# pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #} +{% set integration_key = "so-" ~ integration_type ~ pkg.name + '_x_' ~ component_name_x %} +{# Default integration settings #} +{% set integration_defaults = { + "index_sorting": false, + "index_template": { + "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"], + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"], + "index_patterns": [index_pattern], + "priority": 501, + "template": { + "settings": { + "index": { + "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"}, + "number_of_replicas": 0 + } + } + } + }, + "policy": { + "phases": { + "cold": { + "actions": { + "allocate":{ + "number_of_replicas": "" + }, + "set_priority": {"priority": 0} + }, + "min_age": "60d" + }, + "delete": { + "actions": { + "delete": {} + }, + "min_age": "365d" + }, + "hot": { + "actions": { + "rollover": { + "max_age": "30d", + "max_primary_shard_size": "50gb" + }, + "forcemerge":{ + "max_num_segments": "" + }, + "shrink":{ + "max_primary_shard_size": "", + "method": "COUNT", + "number_of_shards": "" + }, + "set_priority": {"priority": 100} + }, + "min_age": "0ms" + }, + "warm": { + "actions": { + "allocate": { + "number_of_replicas": "" + }, + "forcemerge": { + "max_num_segments": "" + }, + "shrink":{ + "max_primary_shard_size": "", + "method": "COUNT", + "number_of_shards": "" + }, + "set_priority": {"priority": 50} + }, + "min_age": "30d" + } + } + } + } %} + + +{% do ADDON_INPUT_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %} +{% do DEBUG_STUFF.update({integration_key: "Generating defaults for "+ pkg.name })%} +{% endfor %} +{% endif %} +{% endif %} +{% endfor %} diff --git a/salt/elasticfleet/integration-defaults.map.jinja b/salt/elasticfleet/integration-defaults.map.jinja index f85a95ec9..eeb85123a 100644 --- a/salt/elasticfleet/integration-defaults.map.jinja +++ b/salt/elasticfleet/integration-defaults.map.jinja @@ -59,8 +59,8 @@ {# skip core integrations #} {% elif pkg.name not in CORE_ESFLEET_PACKAGES %} {# generate defaults for each integration #} -{% if pkg.es_index_patterns is defined and pkg.es_index_patterns is not none %} -{% for pattern in pkg.es_index_patterns %} +{% if pkg.dataStreams is defined and pkg.dataStreams is not none and pkg.dataStreams | length > 0 %} +{% for pattern in pkg.dataStreams %} {% if "metrics-" in pattern.name %} {% set integration_type = "metrics-" %} {% elif "logs-" in pattern.name %} @@ -75,44 +75,27 @@ {% if component_name in WEIRD_INTEGRATIONS %} {% set component_name = WEIRD_INTEGRATIONS[component_name] %} {% endif %} - -{# create duplicate of component_name, so we can split generics from @custom component templates in the index template below and overwrite the default @package when needed - eg. having to replace unifiedlogs.generic@package with filestream.generic@package, but keep the ability to customize unifiedlogs.generic@custom and its ILM policy #} -{% set custom_component_name = component_name %} - -{# duplicate integration_type to assist with sometimes needing to overwrite component templates with 'logs-filestream.generic@package' (there is no metrics-filestream.generic@package) #} -{% set generic_integration_type = integration_type %} - {# component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #} {% set component_name_x = component_name.replace(".","_x_") %} {# pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #} {% set integration_key = "so-" ~ integration_type ~ component_name_x %} -{# if its a .generic template make sure that a .generic@package for the integration exists. Else default to logs-filestream.generic@package #} -{% if ".generic" in component_name and integration_type ~ component_name ~ "@package" not in INSTALLED_COMPONENT_TEMPLATES %} -{# these generic templates by default are directed to index_pattern of 'logs-generic-*', overwrite that here to point to eg gcp_pubsub.generic-* #} -{% set index_pattern = integration_type ~ component_name ~ "-*" %} -{# includes use of .generic component template, but it doesn't exist in installed component templates. Redirect it to filestream.generic@package #} -{% set component_name = "filestream.generic" %} -{% set generic_integration_type = "logs-" %} -{% endif %} - {# Default integration settings #} {% set integration_defaults = { "index_sorting": false, "index_template": { - "composed_of": [generic_integration_type ~ component_name ~ "@package", integration_type ~ custom_component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"], + "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"], "data_stream": { "allow_custom_routing": false, "hidden": false }, - "ignore_missing_component_templates": [integration_type ~ custom_component_name ~ "@custom"], + "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"], "index_patterns": [index_pattern], "priority": 501, "template": { "settings": { "index": { - "lifecycle": {"name": "so-" ~ integration_type ~ custom_component_name ~ "-logs"}, + "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"}, "number_of_replicas": 0 } } diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common index 1a597b1db..92532082a 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common @@ -135,9 +135,33 @@ elastic_fleet_bulk_package_install() { fi } -elastic_fleet_installed_packages() { - if ! fleet_api "epm/packages/installed?perPage=500"; then +elastic_fleet_get_package_list_by_type() { + if ! output=$(fleet_api "epm/packages"); then return 1 + else + is_integration=$(jq '[.items[] | select(.type=="integration") | .name ]' <<< "$output") + is_input=$(jq '[.items[] | select(.type=="input") | .name ]' <<< "$output") + is_content=$(jq '[.items[] | select(.type=="content") | .name ]' <<< "$output") + jq -n --argjson is_integration "${is_integration:-[]}" \ + --argjson is_input "${is_input:-[]}" \ + --argjson is_content "${is_content:-[]}" \ + '{"integration": $is_integration,"input": $is_input, "content": $is_content}' + fi +} +elastic_fleet_installed_packages_components() { + package_type=${1,,} + if [[ "$package_type" != "integration" && "$package_type" != "input" && "$package_type" != "content" ]]; then + echo "Error: Invalid package type ${package_type}. Valid types are 'integration', 'input', or 'content'." + return 1 + fi + + packages_by_type=$(elastic_fleet_get_package_list_by_type) + packages=$(jq --arg package_type "$package_type" '.[$package_type]' <<< "$packages_by_type") + + if ! output=$(fleet_api "epm/packages/installed?perPage=500"); then + return 1 + else + jq -c --argjson packages "$packages" '[.items[] | select(.name | IN($packages[])) | {name: .name, dataStreams: .dataStreams}]' <<< "$output" fi } diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load index 8c0f627ef..ab38b7065 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load @@ -18,7 +18,9 @@ INSTALLED_PACKAGE_LIST=/tmp/esfleet_installed_packages.json BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json -PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json +INTEGRATION_PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json +INPUT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_input_package_components.json +CONTENT_PACKAGE_COMPONENTS=/opt/so/state/esfleet_content_package_components.json COMPONENT_TEMPLATES=/opt/so/state/esfleet_component_templates.json PENDING_UPDATE=false @@ -179,10 +181,13 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then else echo "Elastic integrations don't appear to need installation/updating..." fi - # Write out file for generating index/component/ilm templates - if latest_installed_package_list=$(elastic_fleet_installed_packages); then - echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS - fi + # Write out file for generating index/component/ilm templates, keeping each package type separate + for package_type in "INTEGRATION" "INPUT" "CONTENT"; do + if latest_installed_package_list=$(elastic_fleet_installed_packages_components "$package_type"); then + outfile="${package_type}_PACKAGE_COMPONENTS" + echo $latest_installed_package_list > "${!outfile}" + fi + done if retry 3 1 "so-elasticsearch-query / --fail --output /dev/null"; then # Refresh installed component template list latest_component_templates_list=$(so-elasticsearch-query _component_template | jq '.component_templates[] | .name' | jq -s '.') diff --git a/salt/elasticsearch/config.sls b/salt/elasticsearch/config.sls index 41ef02164..ac9fa8f72 100644 --- a/salt/elasticsearch/config.sls +++ b/salt/elasticsearch/config.sls @@ -91,6 +91,13 @@ estemplatedir: - group: 939 - makedirs: True +esaddontemplatedir: + file.directory: + - name: /opt/so/conf/elasticsearch/templates/addon-index + - user: 930 + - group: 939 + - makedirs: True + esrolesdir: file.directory: - name: /opt/so/conf/elasticsearch/roles diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index 29ab80329..61b9bad01 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -10,8 +10,7 @@ {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} -{% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} -{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %} +{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, ALL_ADDON_SETTINGS, SO_MANAGED_INDICES %} include: - ca @@ -117,16 +116,29 @@ escomponenttemplates: - onchanges_in: - file: so-elasticsearch-templates-reload - show_changes: False - -# Auto-generate templates from defaults file + +# Clean up legacy and non-SO managed templates from the elasticsearch/templates/index/ directory +so_index_template_dir: + file.directory: + - name: /opt/so/conf/elasticsearch/templates/index + - clean: True + {%- if SO_MANAGED_INDICES %} + - require: + {%- for index in SO_MANAGED_INDICES %} + - file: so_index_template_{{index}} + {%- endfor %} + {%- endif %} + +# Auto-generate index templates for SO managed indices (directly defined in elasticsearch/defaults.yaml) +# These index templates are for the core SO datasets and are always required {% for index, settings in ES_INDEX_SETTINGS.items() %} - {% if settings.index_template is defined %} -es_index_template_{{index}}: +{% if settings.index_template is defined %} +so_index_template_{{index}}: file.managed: - name: /opt/so/conf/elasticsearch/templates/index/{{ index }}-template.json - source: salt://elasticsearch/base-template.json.jinja - defaults: - TEMPLATE_CONFIG: {{ settings.index_template }} + TEMPLATE_CONFIG: {{ settings.index_template }} - template: jinja - show_changes: False - onchanges_in: @@ -134,25 +146,23 @@ es_index_template_{{index}}: {% endif %} {% endfor %} -{% if TEMPLATES %} -# Sync custom templates to /opt/so/conf/elasticsearch/templates -{% for TEMPLATE in TEMPLATES %} -es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}: +# Auto-generate optional index templates for integration | input | content packages +# These index templates are not used by default (until user adds package to an agent policy). +# Pre-configured with standard defaults, and incorporated into SOC configuration for user customization. +{% for index,settings in ALL_ADDON_SETTINGS.items() %} +{% if settings.index_template is defined %} +addon_index_template_{{index}}: file.managed: - - source: salt://elasticsearch/templates/index/{{TEMPLATE}} -{% if 'jinja' in TEMPLATE.split('.')[-1] %} - - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}} + - name: /opt/so/conf/elasticsearch/templates/addon-index/{{ index }}-template.json + - source: salt://elasticsearch/base-template.json.jinja + - defaults: + TEMPLATE_CONFIG: {{ settings.index_template }} - template: jinja -{% else %} - - name: /opt/so/conf/elasticsearch/templates/index/{{TEMPLATE.split('/')[1]}} -{% endif %} - - user: 930 - - group: 939 - show_changes: False - onchanges_in: - - file: so-elasticsearch-templates-reload -{% endfor %} -{% endif %} + - file: addon-elasticsearch-templates-reload +{% endif %} +{% endfor %} {% if GLOBALS.role in GLOBALS.manager_roles %} so-es-cluster-settings: @@ -179,6 +189,10 @@ so-elasticsearch-templates-reload: file.absent: - name: /opt/so/state/estemplates.txt +addon-elasticsearch-templates-reload: + file.absent: + - name: /opt/so/state/addon_estemplates.txt + so-elasticsearch-templates: cmd.run: - name: /usr/sbin/so-elasticsearch-templates-load diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index 2563f8e23..2690fa56f 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -15,14 +15,40 @@ {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %} {# start generation of integration default index_settings #} -{% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') and salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %} -{% set check_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %} -{% if check_package_components.size > 1 %} -{% from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %} -{% for index, settings in ADDON_INTEGRATION_DEFAULTS.items() %} -{% do ES_INDEX_SETTINGS_ORIG.update({index: settings}) %} -{% endfor %} -{% endif%} +{% if salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %} +{% set ALL_ADDON_INTEGRATION_DEFAULTS = {} %} +{% set ALL_ADDON_SETTINGS_ORIG = {} %} +{% set ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES = {} %} +{# import integration type defaults #} +{% if salt['file.file_exists']('/opt/so/state/esfleet_integration_package_components.json') %} +{% set check_integration_package_components = salt['file.stats']('/opt/so/state/esfleet_integration_package_components.json') %} +{% if check_integration_package_components.size > 1 %} +{% from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %} +{% do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_INTEGRATION_DEFAULTS) %} +{% endif %} +{% endif %} + +{# import input type defaults #} +{% if salt['file.file_exists']('/opt/so/state/esfleet_input_package_components.json') %} +{% set check_input_package_components = salt['file.stats']('/opt/so/state/esfleet_input_package_components.json') %} +{% if check_input_package_components.size > 1 %} +{% from 'elasticfleet/input-defaults.map.jinja' import ADDON_INPUT_INTEGRATION_DEFAULTS %} +{% do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_INPUT_INTEGRATION_DEFAULTS) %} +{% endif %} +{% endif %} + +{# import content type defaults #} +{% if salt['file.file_exists']('/opt/so/state/esfleet_content_package_components.json') %} +{% set check_content_package_components = salt['file.stats']('/opt/so/state/esfleet_content_package_components.json') %} +{% if check_content_package_components.size > 1 %} +{% from 'elasticfleet/content-defaults.map.jinja' import ADDON_CONTENT_INTEGRATION_DEFAULTS %} +{% do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_CONTENT_INTEGRATION_DEFAULTS) %} +{% endif %} +{% endif %} + +{% for index, settings in ALL_ADDON_INTEGRATION_DEFAULTS.items() %} +{% do ALL_ADDON_SETTINGS_ORIG.update({index: settings}) %} +{% endfor %} {% endif %} {# end generation of integration default index_settings #} @@ -31,25 +57,34 @@ {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ELASTICSEARCHDEFAULTS.elasticsearch.index_settings[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %} {% endfor %} +{% if ALL_ADDON_SETTINGS_ORIG.keys() | length > 0 %} +{% for index in ALL_ADDON_SETTINGS_ORIG.keys() %} +{% do ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ALL_ADDON_SETTINGS_ORIG[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %} +{% endfor %} +{% endif %} + {% set ES_INDEX_SETTINGS = {} %} -{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %} -{% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %} +{% set ALL_ADDON_SETTINGS = {}%} +{% macro create_final_index_template(DEFINED_SETTINGS, GLOBAL_OVERRIDES, FINAL_INDEX_SETTINGS) %} + +{% do GLOBAL_OVERRIDES.update(salt['defaults.merge'](GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %} +{% for index, settings in GLOBAL_OVERRIDES.items() %} {# prevent this action from being performed on custom defined indices. #} {# the custom defined index is not present in either of the dictionaries and fails to reder. #} -{% if index in ES_INDEX_SETTINGS_ORIG and index in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES %} +{% if index in DEFINED_SETTINGS and index in GLOBAL_OVERRIDES %} {# dont merge policy from the global_overrides if policy isn't defined in the original index settingss #} {# this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #} -{% if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %} -{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %} +{% if not DEFINED_SETTINGS[index].policy is defined and GLOBAL_OVERRIDES[index].policy is defined %} +{% do GLOBAL_OVERRIDES[index].pop('policy') %} {% endif %} {# this prevents and index from inderiting a policy phase from global overrides if it wasnt defined in the defaults. #} -{% if ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %} -{% for phase in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.copy() %} -{% if ES_INDEX_SETTINGS_ORIG[index].policy.phases[phase] is not defined %} -{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %} +{% if GLOBAL_OVERRIDES[index].policy is defined %} +{% for phase in GLOBAL_OVERRIDES[index].policy.phases.copy() %} +{% if DEFINED_SETTINGS[index].policy.phases[phase] is not defined %} +{% do GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %} {% endif %} {% endfor %} {% endif %} @@ -111,5 +146,14 @@ {% endfor %} {% endif %} -{% do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %} +{% do FINAL_INDEX_SETTINGS.update({index | replace("_x_", "."): GLOBAL_OVERRIDES[index]}) %} {% endfor %} +{% endmacro %} + +{{ create_final_index_template(ES_INDEX_SETTINGS_ORIG, ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_SETTINGS) }} +{{ create_final_index_template(ALL_ADDON_SETTINGS_ORIG, ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES, ALL_ADDON_SETTINGS) }} + +{% set SO_MANAGED_INDICES = [] %} +{% for index, settings in ES_INDEX_SETTINGS.items() %} +{% do SO_MANAGED_INDICES.append(index) %} +{% endfor %} \ No newline at end of file From 6b8a6267daab3875d3288d2eeadbaf80138c5467 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 9 Apr 2026 16:45:26 -0500 Subject: [PATCH 09/24] remove unused elasticsearch:index_template pillar references --- pillar/elasticsearch/index_templates.sls | 2 -- pillar/top.sls | 3 --- 2 files changed, 5 deletions(-) delete mode 100644 pillar/elasticsearch/index_templates.sls diff --git a/pillar/elasticsearch/index_templates.sls b/pillar/elasticsearch/index_templates.sls deleted file mode 100644 index a02a1818c..000000000 --- a/pillar/elasticsearch/index_templates.sls +++ /dev/null @@ -1,2 +0,0 @@ -elasticsearch: - index_settings: diff --git a/pillar/top.sls b/pillar/top.sls index 6cdc4808a..d3b24677c 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -97,7 +97,6 @@ base: - node_data.ips - secrets - healthcheck.eval - - elasticsearch.index_templates {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} @@ -142,7 +141,6 @@ base: - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash - - elasticsearch.index_templates {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} @@ -256,7 +254,6 @@ base: '*_import': - node_data.ips - secrets - - elasticsearch.index_templates {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} From 378d1ec81b68d99e0596e07657b1c335a1124dcb Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 9 Apr 2026 18:41:40 -0500 Subject: [PATCH 10/24] initialize vars --- salt/elasticsearch/template.map.jinja | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index 2690fa56f..fc510324a 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -14,11 +14,12 @@ {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %} +{% set ALL_ADDON_INTEGRATION_DEFAULTS = {} %} +{% set ALL_ADDON_SETTINGS_ORIG = {} %} +{% set ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES = {} %} +{% set ALL_ADDON_SETTINGS = {} %} {# start generation of integration default index_settings #} {% if salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %} -{% set ALL_ADDON_INTEGRATION_DEFAULTS = {} %} -{% set ALL_ADDON_SETTINGS_ORIG = {} %} -{% set ALL_ADDON_SETTINGS_GLOBAL_OVERRIDES = {} %} {# import integration type defaults #} {% if salt['file.file_exists']('/opt/so/state/esfleet_integration_package_components.json') %} {% set check_integration_package_components = salt['file.stats']('/opt/so/state/esfleet_integration_package_components.json') %} @@ -64,7 +65,6 @@ {% endif %} {% set ES_INDEX_SETTINGS = {} %} -{% set ALL_ADDON_SETTINGS = {}%} {% macro create_final_index_template(DEFINED_SETTINGS, GLOBAL_OVERRIDES, FINAL_INDEX_SETTINGS) %} {% do GLOBAL_OVERRIDES.update(salt['defaults.merge'](GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %} From 6298397534c1c76fa54ed5b54f7853d797066a26 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Sat, 11 Apr 2026 04:40:47 -0500 Subject: [PATCH 11/24] rework elasticsearch template load script -- for core templates --- salt/elasticsearch/enabled.sls | 20 +- .../so-elasticsearch-component-templates-list | 19 +- .../sbin/so-elasticsearch-templates-load | 190 ++++++++++++++++++ .../so-elasticsearch-templates-load | 165 --------------- 4 files changed, 219 insertions(+), 175 deletions(-) create mode 100755 salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load delete mode 100755 salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index 61b9bad01..d95ec2f98 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -10,7 +10,10 @@ {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} -{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, ALL_ADDON_SETTINGS, SO_MANAGED_INDICES %} +{% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS, SO_MANAGED_INDICES %} +{% if GLOBALS.role != 'so-heavynode' %} +{% from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %} +{% endif %} include: - ca @@ -140,17 +143,17 @@ so_index_template_{{index}}: - defaults: TEMPLATE_CONFIG: {{ settings.index_template }} - template: jinja - - show_changes: False - onchanges_in: - file: so-elasticsearch-templates-reload {% endif %} {% endfor %} +{% if GLOBALS.role != "so-heavynode" %} # Auto-generate optional index templates for integration | input | content packages # These index templates are not used by default (until user adds package to an agent policy). # Pre-configured with standard defaults, and incorporated into SOC configuration for user customization. -{% for index,settings in ALL_ADDON_SETTINGS.items() %} -{% if settings.index_template is defined %} +{% for index,settings in ALL_ADDON_SETTINGS.items() %} +{% if settings.index_template is defined %} addon_index_template_{{index}}: file.managed: - name: /opt/so/conf/elasticsearch/templates/addon-index/{{ index }}-template.json @@ -161,8 +164,9 @@ addon_index_template_{{index}}: - show_changes: False - onchanges_in: - file: addon-elasticsearch-templates-reload -{% endif %} -{% endfor %} +{% endif %} +{% endfor %} +{% endif %} {% if GLOBALS.role in GLOBALS.manager_roles %} so-es-cluster-settings: @@ -195,7 +199,11 @@ addon-elasticsearch-templates-reload: so-elasticsearch-templates: cmd.run: +{%- if GLOBALS.role == "so-heavynode" %} + - name: /usr/sbin/so-elasticsearch-templates-load --heavynode +{%- else %} - name: /usr/sbin/so-elasticsearch-templates-load +{%- endif %} - cwd: /opt/so - template: jinja - require: diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list b/salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list index 2fccce9cb..6946e30da 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list @@ -6,8 +6,19 @@ # Elastic License 2.0. . /usr/sbin/so-common -if [ "$1" == "" ]; then - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template | jq '.component_templates[] |.name'| sort + +if [[ -z "$1" ]]; then + if output=$(so-elasticsearch-query "_component_template" --retry 3 --retry-delay 1 --fail); then + jq '[.component_templates[] | .name] | sort' <<< "$output" + else + echo "Failed to retrieve component templates from Elasticsearch." + exit 1 + fi else - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template/$1 | jq -fi + if output=$(so-elasticsearch-query "_component_template/$1" --retry 3 --retry-delay 1 --fail); then + jq <<< "$output" + else + echo "Failed to retrieve component template '$1' from Elasticsearch." + exit 1 + fi +fi \ No newline at end of file diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load new file mode 100755 index 000000000..f44225ac6 --- /dev/null +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load @@ -0,0 +1,190 @@ +#!/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +SO_STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt +ADDON_STATE_FILE_SUCCESS=/opt/so/state/addon-estemplates.txt +ELASTICSEARCH_TEMPLATES_DIR="/opt/so/conf/elasticsearch/templates" +SO_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/index" +ADDON_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/addon-index" +LOAD_FAILURES=0 +LOAD_FAILURES_NAMES=() +IS_HEAVYNODE="false" +FORCE="false" +VERBOSE="false" + +# If soup is running, ignore errors +pgrep soup >/dev/null && should_exit_on_failure=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --heavynode) + IS_HEAVYNODE="true" + ;; + --force) + FORCE="true" + ;; + --verbose) + VERBOSE="true" + ;; + *) + echo "Usage: $0 [options]" + echo "Options:" + echo " --heavynode Only loads index templates specific to heavynodes" + echo " --force Force reload all templates regardless of state file (default: false)" + echo " --verbose Enable verbose output" + exit 1 + ;; + esac + shift +done + +load_template() { + local uri="$1" + local file="$2" + + echo "Loading template file $file" + if ! output=$(retry 3 3 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"); then + echo "$output" + + return 1 + + elif [[ "$VERBOSE" == "true" ]]; then + echo "$output" + fi + +} + +check_required_component_template_exists() { + local required + local missing + local file=$1 + + required=$(jq '[((.composed_of //[]) - (.ignore_missing_component_templates // []))[]]' "$file") + missing=$(jq -n --argjson required "$required" --argjson component_templates "$component_templates" '(($required) - ($component_templates))') + + if [[ $(jq length <<<"$missing") -gt 0 ]]; then + + return 1 + fi +} + +check_heavynode_compatiable_index_template() { + # The only templates that are relevant to heavynodes are from datasets defined in elasticagent/files/elastic-agent.yml.jinja. + # Heavynodes do not have fleet server packages installed and do not support elastic agents reporting directly to them. + local -A heavynode_index_templates=( + ["so-import"]=1 + ["so-syslog"]=1 + ["so-logs-soc"]=1 + ["so-suricata"]=1 + ["so-suricata.alerts"]=1 + ["so-zeek"]=1 + ["so-strelka"]=1 + ) + + local template_name="$1" + + if [[ ! -v heavynode_index_templates["$template_name"] ]]; then + + return 1 + fi + +} + +load_component_templates() { + local printed_name="$1" + local pattern="${ELASTICSEARCH_TEMPLATES_DIR}/component/$2" + + # current state of nullglob shell option + shopt -q nullglob && nullglob_set=1 || nullglob_set=0 + + shopt -s nullglob + echo -e "\nLoading $printed_name component templates...\n" + for component in "$pattern"/*.json; do + tmpl_name=$(basename "${component%.json}") + if ! load_template "_component_template/${tmpl_name}-mappings" "$component"; then + LOAD_FAILURES=$((LOAD_FAILURES + 1)) + LOAD_FAILURES_NAMES+=("$component") + fi + done + + # restore nullglob shell option if needed + if [[ $nullglob_set -eq 1 ]]; then + shopt -u nullglob + fi +} + +if [[ "$FORCE" == "true" || ! -f "$SO_STATE_FILE_SUCCESS" ]]; then + # Cannot load templates if Elasticsearch is not responding. + # NOTE: Slightly faster exit w/ failure than previous "retry 240 1" if there is a problem with Elasticsearch the + # script should exit sooner rather than hang at the 'so-elasticsearch-templates' salt state. + retry 3 15 "so-elasticsearch-query / --output /dev/null --fail" || + fail "Elasticsearch is not responding. Please review Elasticsearch logs /opt/so/log/elasticsearch/securityonion.log for more details. Additionally, consider running so-elasticsearch-troubleshoot." + + if [[ "$IS_HEAVYNODE" == "false" ]]; then + # TODO: Better way to check if fleet server is installed vs checking for Elastic Defend component template. + fleet_check="logs-endpoint.alerts@package" + if ! so-elasticsearch-query "_component_template/$fleet_check" --output /dev/null --retry 5 --retry-delay 3 --fail; then + echo -e "\nPackage $fleet_check not yet installed. Fleet Server may not be fully configured yet." + # Fleet Server is required because some SO index templates depend on components installed via + # specific integrations eg Elastic Defend. These are components that we do not manually create / manage + # via /opt/so/saltstack/salt/elasticsearch/templates/component/ + + exit 0 + fi + fi + + load_component_templates "ECS" "ecs" + load_component_templates "Elastic Agent" "elastic-agent" + load_component_templates "Security Onion" "securityonion" + + component_templates=$(so-elasticsearch-component-templates-list) + echo -e "Loading Security Onion index templates...\n" + for so_idx_tmpl in "${SO_TEMPLATES_DIR}"/*.json; do + tmpl_name=$(basename "${so_idx_tmpl%-template.json}") + + if [[ "$IS_HEAVYNODE" == "true" ]]; then + # TODO: Better way to load only heavynode specific templates + if ! check_heavynode_compatiable_index_template "$tmpl_name"; then + if [[ "$VERBOSE" == "true" ]]; then + echo "Skipping over $so_idx_tmpl, template is not a heavynode specific index template." + fi + + continue + fi + fi + + if check_required_component_template_exists "$so_idx_tmpl"; then + if ! load_template "_index_template/$tmpl_name" "$so_idx_tmpl"; then + LOAD_FAILURES=$((LOAD_FAILURES + 1)) + LOAD_FAILURES_NAMES+=("$so_idx_tmpl") + fi + else + echo "Skipping over $so_idx_tmpl due to missing required component template(s)." + LOAD_FAILURES=$((LOAD_FAILURES + 1)) + LOAD_FAILURES_NAMES+=("$so_idx_tmpl") + + continue + fi + done + + if [[ $LOAD_FAILURES -eq 0 ]]; then + echo "All templates loaded successfully." + + touch "$SO_STATE_FILE_SUCCESS" + else + echo "Encountered $LOAD_FAILURES failure(s) loading templates:" + for failed_template in "${LOAD_FAILURES_NAMES[@]}"; do + echo " - $failed_template" + done + fi + +else + + echo "Templates already loaded" + +fi \ No newline at end of file diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load deleted file mode 100755 index ad3fe1344..000000000 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ /dev/null @@ -1,165 +0,0 @@ -#!/bin/bash -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. -{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} -{% from 'vars/globals.map.jinja' import GLOBALS %} - -STATE_FILE_INITIAL=/opt/so/state/estemplates_initial_load_attempt.txt -STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt - -if [[ -f $STATE_FILE_INITIAL ]]; then - # The initial template load has already run. As this is a subsequent load, all dependencies should - # already be satisified. Therefore, immediately exit/abort this script upon any template load failure - # since this is an unrecoverable failure. - should_exit_on_failure=1 -else - # This is the initial template load, and there likely are some components not yet setup in Elasticsearch. - # Therefore load as many templates as possible at this time and if an error occurs proceed to the next - # template. But if at least one template fails to load do not mark the templates as having been loaded. - # This will allow the next load to resume the load of the templates that failed to load initially. - should_exit_on_failure=0 - echo "This is the initial template load" -fi - -# If soup is running, ignore errors -pgrep soup > /dev/null && should_exit_on_failure=0 - -load_failures=0 - -load_template() { - uri=$1 - file=$2 - - echo "Loading template file $i" - if ! retry 3 1 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"; then - if [[ $should_exit_on_failure -eq 1 ]]; then - fail "Could not load template file: $file" - else - load_failures=$((load_failures+1)) - echo "Incremented load failure counter: $load_failures" - fi - fi -} - -if [ ! -f $STATE_FILE_SUCCESS ]; then - echo "State file $STATE_FILE_SUCCESS not found. Running so-elasticsearch-templates-load." - - . /usr/sbin/so-common - - {% if GLOBALS.role != 'so-heavynode' %} - if [ -f /usr/sbin/so-elastic-fleet-common ]; then - . /usr/sbin/so-elastic-fleet-common - fi - {% endif %} - - default_conf_dir=/opt/so/conf - - # Define a default directory to load pipelines from - ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/" - - {% if GLOBALS.role == 'so-heavynode' %} - file="/opt/so/conf/elasticsearch/templates/index/so-common-template.json" - {% else %} - file="/usr/sbin/so-elastic-fleet-common" - {% endif %} - - if [ -f "$file" ]; then - # Wait for ElasticSearch to initialize - echo -n "Waiting for ElasticSearch..." - retry 240 1 "so-elasticsearch-query / -k --output /dev/null --silent --head --fail" || fail "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" - {% if GLOBALS.role != 'so-heavynode' %} - TEMPLATE="logs-endpoint.alerts@package" - INSTALLED=$(so-elasticsearch-query _component_template/$TEMPLATE | jq -r .component_templates[0].name) - if [ "$INSTALLED" != "$TEMPLATE" ]; then - echo - echo "Packages not yet installed." - echo - exit 0 - fi - {% endif %} - - touch $STATE_FILE_INITIAL - - cd ${ELASTICSEARCH_TEMPLATES}/component/ecs - - echo "Loading ECS component templates..." - for i in *; do - TEMPLATE=$(echo $i | cut -d '.' -f1) - load_template "_component_template/${TEMPLATE}-mappings" "$i" - done - echo - - cd ${ELASTICSEARCH_TEMPLATES}/component/elastic-agent - - echo "Loading Elastic Agent component templates..." - {% if GLOBALS.role == 'so-heavynode' %} - component_pattern="so-*" - {% else %} - component_pattern="*" - {% endif %} - for i in $component_pattern; do - TEMPLATE=${i::-5} - load_template "_component_template/$TEMPLATE" "$i" - done - echo - - # Load SO-specific component templates - cd ${ELASTICSEARCH_TEMPLATES}/component/so - - echo "Loading Security Onion component templates..." - for i in *; do - TEMPLATE=$(echo $i | cut -d '.' -f1); - load_template "_component_template/$TEMPLATE" "$i" - done - echo - - # Load SO index templates - cd ${ELASTICSEARCH_TEMPLATES}/index - - echo "Loading Security Onion index templates..." - shopt -s extglob - {% if GLOBALS.role == 'so-heavynode' %} - pattern="!(*1password*|*aws*|*azure*|*cloudflare*|*elastic_agent*|*fim*|*github*|*google*|*osquery*|*system*|*windows*|*endpoint*|*elasticsearch*|*generic*|*fleet_server*|*soc*)" - {% else %} - pattern="*" - {% endif %} - # Index templates will be skipped if the following conditions are met: - # 1. The template is part of the "so-logs-" template group - # 2. The template name does not correlate to at least one existing component template - # In this situation, the script will treat the skipped template as a temporary failure - # and allow the templates to be loaded again on the next run or highstate, whichever - # comes first. - COMPONENT_LIST=$(so-elasticsearch-component-templates-list) - for i in $pattern; do - TEMPLATE=${i::-14} - COMPONENT_PATTERN=${TEMPLATE:3} - MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery") - if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ \.generic|logs-winlog\.winlog ]]; then - load_failures=$((load_failures+1)) - echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures" - else - load_template "_index_template/$TEMPLATE" "$i" - fi - done - else - {% if GLOBALS.role == 'so-heavynode' %} - echo "Common template does not exist. Exiting..." - {% else %} - echo "Elastic Fleet not configured. Exiting..." - {% endif %} - exit 0 - fi - - cd - >/dev/null - - if [[ $load_failures -eq 0 ]]; then - echo "All templates loaded successfully" - touch $STATE_FILE_SUCCESS - else - echo "Encountered $load_failures templates that were unable to load, likely due to missing dependencies that will be available later; will retry on next highstate" - fi -else - echo "Templates already loaded" -fi From b0584a4dc5358b75ce7c50564328a7a280fe2059 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Sat, 11 Apr 2026 15:22:50 -0500 Subject: [PATCH 12/24] only append "-mappings" to component template names as needed --- .../tools/sbin/so-elasticsearch-templates-load | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load index f44225ac6..3b5aa3707 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load @@ -98,6 +98,7 @@ check_heavynode_compatiable_index_template() { load_component_templates() { local printed_name="$1" local pattern="${ELASTICSEARCH_TEMPLATES_DIR}/component/$2" + local append_mappings="${3:-"false"}" # current state of nullglob shell option shopt -q nullglob && nullglob_set=1 || nullglob_set=0 @@ -106,7 +107,13 @@ load_component_templates() { echo -e "\nLoading $printed_name component templates...\n" for component in "$pattern"/*.json; do tmpl_name=$(basename "${component%.json}") - if ! load_template "_component_template/${tmpl_name}-mappings" "$component"; then + + if [[ "$append_mappings" == "true" ]]; then + # avoid duplicating "-mappings" if it already exists in the component template filename + tmpl_name="${tmpl_name%-mappings}-mappings" + fi + + if ! load_template "_component_template/${tmpl_name}" "$component"; then LOAD_FAILURES=$((LOAD_FAILURES + 1)) LOAD_FAILURES_NAMES+=("$component") fi @@ -138,9 +145,9 @@ if [[ "$FORCE" == "true" || ! -f "$SO_STATE_FILE_SUCCESS" ]]; then fi fi - load_component_templates "ECS" "ecs" + load_component_templates "ECS" "ecs" "true" load_component_templates "Elastic Agent" "elastic-agent" - load_component_templates "Security Onion" "securityonion" + load_component_templates "Security Onion" "so" component_templates=$(so-elasticsearch-component-templates-list) echo -e "Loading Security Onion index templates...\n" @@ -187,4 +194,4 @@ else echo "Templates already loaded" -fi \ No newline at end of file +fi From a43947cca5649f77ef0b3436efae33bd92a0da3b Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Sun, 12 Apr 2026 00:23:26 -0500 Subject: [PATCH 13/24] elasticsearch template load script -- for addon index templates --- salt/elasticsearch/enabled.sls | 1 + salt/elasticsearch/template.map.jinja | 4 +- .../sbin/so-elasticsearch-templates-load | 91 +++++++++++++++---- 3 files changed, 74 insertions(+), 22 deletions(-) diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index d95ec2f98..dc404c509 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -197,6 +197,7 @@ addon-elasticsearch-templates-reload: file.absent: - name: /opt/so/state/addon_estemplates.txt +# so-elasticsearch-templates-load will have its first successful run during the 'so-elastic-fleet-setup' script so-elasticsearch-templates: cmd.run: {%- if GLOBALS.role == "so-heavynode" %} diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index fc510324a..e66057775 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -21,8 +21,8 @@ {# start generation of integration default index_settings #} {% if salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %} {# import integration type defaults #} -{% if salt['file.file_exists']('/opt/so/state/esfleet_integration_package_components.json') %} -{% set check_integration_package_components = salt['file.stats']('/opt/so/state/esfleet_integration_package_components.json') %} +{% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') %} +{% set check_integration_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %} {% if check_integration_package_components.size > 1 %} {% from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %} {% do ALL_ADDON_INTEGRATION_DEFAULTS.update(ADDON_INTEGRATION_DEFAULTS) %} diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load index 3b5aa3707..3a0361a9f 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load @@ -6,19 +6,22 @@ . /usr/sbin/so-common -SO_STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt -ADDON_STATE_FILE_SUCCESS=/opt/so/state/addon-estemplates.txt +SO_STATEFILE_SUCCESS=/opt/so/state/estemplates.txt +ADDON_STATEFILE_SUCCESS=/opt/so/state/addon-estemplates.txt ELASTICSEARCH_TEMPLATES_DIR="/opt/so/conf/elasticsearch/templates" SO_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/index" ADDON_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/addon-index" -LOAD_FAILURES=0 -LOAD_FAILURES_NAMES=() +SO_LOAD_FAILURES=0 +ADDON_LOAD_FAILURES=0 +SO_LOAD_FAILURES_NAMES=() +ADDON_LOAD_FAILURES_NAMES=() IS_HEAVYNODE="false" FORCE="false" VERBOSE="false" +SHOULD_EXIT_ON_FAILURE="true" # If soup is running, ignore errors -pgrep soup >/dev/null && should_exit_on_failure=0 +pgrep soup >/dev/null && SHOULD_EXIT_ON_FAILURE="false" while [[ $# -gt 0 ]]; do case "$1" in @@ -35,7 +38,7 @@ while [[ $# -gt 0 ]]; do echo "Usage: $0 [options]" echo "Options:" echo " --heavynode Only loads index templates specific to heavynodes" - echo " --force Force reload all templates regardless of state file (default: false)" + echo " --force Force reload all templates regardless of statefiles (default: false)" echo " --verbose Enable verbose output" exit 1 ;; @@ -114,8 +117,8 @@ load_component_templates() { fi if ! load_template "_component_template/${tmpl_name}" "$component"; then - LOAD_FAILURES=$((LOAD_FAILURES + 1)) - LOAD_FAILURES_NAMES+=("$component") + SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1)) + SO_LOAD_FAILURES_NAMES+=("$component") fi done @@ -125,17 +128,22 @@ load_component_templates() { fi } -if [[ "$FORCE" == "true" || ! -f "$SO_STATE_FILE_SUCCESS" ]]; then +check_elasticsearch_responsive() { # Cannot load templates if Elasticsearch is not responding. # NOTE: Slightly faster exit w/ failure than previous "retry 240 1" if there is a problem with Elasticsearch the # script should exit sooner rather than hang at the 'so-elasticsearch-templates' salt state. retry 3 15 "so-elasticsearch-query / --output /dev/null --fail" || fail "Elasticsearch is not responding. Please review Elasticsearch logs /opt/so/log/elasticsearch/securityonion.log for more details. Additionally, consider running so-elasticsearch-troubleshoot." +} + +if [[ "$FORCE" == "true" || ! -f "$SO_STATEFILE_SUCCESS" ]]; then + check_elasticsearch_responsive if [[ "$IS_HEAVYNODE" == "false" ]]; then # TODO: Better way to check if fleet server is installed vs checking for Elastic Defend component template. fleet_check="logs-endpoint.alerts@package" if ! so-elasticsearch-query "_component_template/$fleet_check" --output /dev/null --retry 5 --retry-delay 3 --fail; then + # This check prevents so-elasticsearch-templates-load from running before so-elastic-fleet-setup has run. echo -e "\nPackage $fleet_check not yet installed. Fleet Server may not be fully configured yet." # Fleet Server is required because some SO index templates depend on components installed via # specific integrations eg Elastic Defend. These are components that we do not manually create / manage @@ -145,6 +153,7 @@ if [[ "$FORCE" == "true" || ! -f "$SO_STATE_FILE_SUCCESS" ]]; then fi fi + # load_component_templates "Name" "directory" "append '-mappings'?" load_component_templates "ECS" "ecs" "true" load_component_templates "Elastic Agent" "elastic-agent" load_component_templates "Security Onion" "so" @@ -167,31 +176,73 @@ if [[ "$FORCE" == "true" || ! -f "$SO_STATE_FILE_SUCCESS" ]]; then if check_required_component_template_exists "$so_idx_tmpl"; then if ! load_template "_index_template/$tmpl_name" "$so_idx_tmpl"; then - LOAD_FAILURES=$((LOAD_FAILURES + 1)) - LOAD_FAILURES_NAMES+=("$so_idx_tmpl") + SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1)) + SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl") fi else echo "Skipping over $so_idx_tmpl due to missing required component template(s)." - LOAD_FAILURES=$((LOAD_FAILURES + 1)) - LOAD_FAILURES_NAMES+=("$so_idx_tmpl") + SO_LOAD_FAILURES=$((SO_LOAD_FAILURES + 1)) + SO_LOAD_FAILURES_NAMES+=("$so_idx_tmpl") continue fi done - if [[ $LOAD_FAILURES -eq 0 ]]; then - echo "All templates loaded successfully." + if [[ $SO_LOAD_FAILURES -eq 0 ]]; then + echo "All Security Onion core templates loaded successfully." - touch "$SO_STATE_FILE_SUCCESS" + touch "$SO_STATEFILE_SUCCESS" else - echo "Encountered $LOAD_FAILURES failure(s) loading templates:" - for failed_template in "${LOAD_FAILURES_NAMES[@]}"; do + echo "Encountered $SO_LOAD_FAILURES failure(s) loading templates:" + for failed_template in "${SO_LOAD_FAILURES_NAMES[@]}"; do echo " - $failed_template" done + if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then + fail "Failed to load all Security Onion core templates successfully." + fi fi - else - echo "Templates already loaded" + echo "Security Onion core templates already loaded" +fi + +# Start loading addon templates +if [[ (-f "$SO_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" && ! -f "$ADDON_STATEFILE_SUCCESS") || "$FORCE" == "true" ]]; then + + check_elasticsearch_responsive + + echo -e "\nLoading addon integration index templates...\n" + component_templates=$(so-elasticsearch-component-templates-list) + + for addon_idx_tmpl in "${ADDON_TEMPLATES_DIR}"/*.json; do + tmpl_name=$(basename "${addon_idx_tmpl%-template.json}") + + if check_required_component_template_exists "$addon_idx_tmpl"; then + if ! load_template "_index_template/${tmpl_name}" "$addon_idx_tmpl"; then + ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1)) + ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl") + fi + else + echo "Skipping over $addon_idx_tmpl due to missing required component template(s)." + ADDON_LOAD_FAILURES=$((ADDON_LOAD_FAILURES + 1)) + ADDON_LOAD_FAILURES_NAMES+=("$addon_idx_tmpl") + + continue + fi + done + + if [[ $ADDON_LOAD_FAILURES -eq 0 ]]; then + echo "All addon integration templates loaded successfully." + + touch "$ADDON_STATEFILE_SUCCESS" + else + echo "Encountered $ADDON_LOAD_FAILURES failure(s) loading addon integration templates:" + for failed_template in "${ADDON_LOAD_FAILURES_NAMES[@]}"; do + echo " - $failed_template" + done + if [[ "$SHOULD_EXIT_ON_FAILURE" == "true" ]]; then + fail "Failed to load all addon integration templates successfully." + fi + fi fi From abcad9fde04aedf0080b943c298bcacfb00e64d6 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Sun, 12 Apr 2026 00:36:30 -0500 Subject: [PATCH 14/24] addon statefile --- salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load index 3a0361a9f..e9b382cba 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load @@ -7,7 +7,7 @@ . /usr/sbin/so-common SO_STATEFILE_SUCCESS=/opt/so/state/estemplates.txt -ADDON_STATEFILE_SUCCESS=/opt/so/state/addon-estemplates.txt +ADDON_STATEFILE_SUCCESS=/opt/so/state/addon_estemplates.txt ELASTICSEARCH_TEMPLATES_DIR="/opt/so/conf/elasticsearch/templates" SO_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/index" ADDON_TEMPLATES_DIR="${ELASTICSEARCH_TEMPLATES_DIR}/addon-index" From 29e13b2c0b282418ab3d53761c949f1df0c8e727 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 13 Apr 2026 10:00:17 -0500 Subject: [PATCH 15/24] elasticsearch ilm policy load script --- salt/elasticsearch/config.sls | 2 ++ salt/elasticsearch/enabled.sls | 1 + .../sbin_jinja/so-elasticsearch-ilm-policy-load | 13 +++++++++++++ 3 files changed, 16 insertions(+) diff --git a/salt/elasticsearch/config.sls b/salt/elasticsearch/config.sls index ac9fa8f72..8a4674c71 100644 --- a/salt/elasticsearch/config.sls +++ b/salt/elasticsearch/config.sls @@ -66,6 +66,8 @@ so-elasticsearch-ilm-policy-load-script: - group: 939 - mode: 754 - template: jinja + - defaults: + GLOBALS: {{ GLOBALS }} - show_changes: False so-elasticsearch-pipelines-script: diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index dc404c509..f4031ee5d 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -179,6 +179,7 @@ so-es-cluster-settings: - file: elasticsearch_sbin_jinja {% endif %} +# heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml) so-elasticsearch-ilm-policy-load: cmd.run: - name: /usr/sbin/so-elasticsearch-ilm-policy-load diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load index 04a7a8ab0..7988c1905 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load @@ -7,6 +7,9 @@ . /usr/sbin/so-common {%- from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %} +{%- if GLOBALS.role != "so-heavynode" %} +{%- from 'elasticsearch/template.map.jinja' import ALL_ADDON_SETTINGS %} +{%- endif %} {%- for index, settings in ES_INDEX_SETTINGS.items() %} {%- if settings.policy is defined %} @@ -33,3 +36,13 @@ {%- endif %} {%- endfor %} echo +{%- if GLOBALS.role != "so-heavynode" %} +{%- for index, settings in ALL_ADDON_SETTINGS.items() %} +{%- if settings.policy is defined %} + echo + echo "Setting up {{ index }}-logs policy..." + curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' + echo +{%- endif %} +{%- endfor %} +{%- endif %} From dd40e44530faca3e4849cbd08062d7960df4227d Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 13 Apr 2026 12:36:42 -0500 Subject: [PATCH 16/24] show when addon integrations are already loaded --- .../tools/sbin/so-elasticsearch-templates-load | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load index e9b382cba..9ad0f418a 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load @@ -207,7 +207,7 @@ else fi # Start loading addon templates -if [[ (-f "$SO_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" && ! -f "$ADDON_STATEFILE_SUCCESS") || "$FORCE" == "true" ]]; then +if [[ (-f "$SO_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" && ! -f "$ADDON_STATEFILE_SUCCESS") || ("$IS_HEAVYNODE" == "false" && "$FORCE" == "true") ]]; then check_elasticsearch_responsive @@ -245,4 +245,9 @@ if [[ (-f "$SO_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" && ! -f "$ADDON_ fi fi +elif [[ ! -f "$SO_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" ]]; then + echo "Skipping loading addon integration templates until Security Onion core templates have been loaded." + +elif [[ -f "$ADDON_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" && "$FORCE" == "false" ]]; then + echo "Addon integration templates already loaded" fi From a232cd89cc1fbdc5473fb685d1e1bc541c1dd1e1 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 13 Apr 2026 13:36:51 -0500 Subject: [PATCH 17/24] ES 9.3.3 --- .../grid-nodes_general/import-evtx-logs.json | 2 +- salt/elasticsearch/defaults.yaml | 2 +- ...nse.log-1.25.1 => logs-pfsense.log-1.25.2} | 24 +++++++++---------- ...icata => logs-pfsense.log-1.25.2-suricata} | 0 4 files changed, 14 insertions(+), 14 deletions(-) rename salt/elasticsearch/files/ingest/{logs-pfsense.log-1.25.1 => logs-pfsense.log-1.25.2} (94%) rename salt/elasticsearch/files/ingest/{logs-pfsense.log-1.25.1-suricata => logs-pfsense.log-1.25.2-suricata} (100%) diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json index 0e42a0dfb..32d210172 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json @@ -29,7 +29,7 @@ "\\.gz$" ], "include_files": [], - "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.13.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.6.0\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.13.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.13.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.6.0\n- add_fields:\n target: data_stream\n fields:\n dataset: import", + "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.15.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.8.0\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.15.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.15.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.8.0\n- add_fields:\n target: data_stream\n fields:\n dataset: import", "tags": [ "import" ], diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index f355601dc..6fb795bce 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1,6 +1,6 @@ elasticsearch: enabled: false - version: 9.3.2 + version: 9.3.3 index_clean: true vm: max_map_count: 1048576 diff --git a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.1 b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2 similarity index 94% rename from salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.1 rename to salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2 index 3037ce77a..1ea828514 100644 --- a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.1 +++ b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2 @@ -118,77 +118,77 @@ { "pipeline": { "tag": "pipeline_e16851a7", - "name": "logs-pfsense.log-1.25.1-firewall", + "name": "logs-pfsense.log-1.25.2-firewall", "if": "ctx.event.provider == 'filterlog'" } }, { "pipeline": { "tag": "pipeline_828590b5", - "name": "logs-pfsense.log-1.25.1-openvpn", + "name": "logs-pfsense.log-1.25.2-openvpn", "if": "ctx.event.provider == 'openvpn'" } }, { "pipeline": { "tag": "pipeline_9d37039c", - "name": "logs-pfsense.log-1.25.1-ipsec", + "name": "logs-pfsense.log-1.25.2-ipsec", "if": "ctx.event.provider == 'charon'" } }, { "pipeline": { "tag": "pipeline_ad56bbca", - "name": "logs-pfsense.log-1.25.1-dhcp", - "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\"].contains(ctx.event.provider)" + "name": "logs-pfsense.log-1.25.2-dhcp", + "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\", \"dnsmasq-dhcp\"].contains(ctx.event.provider)" } }, { "pipeline": { "tag": "pipeline_dd85553d", - "name": "logs-pfsense.log-1.25.1-unbound", + "name": "logs-pfsense.log-1.25.2-unbound", "if": "ctx.event.provider == 'unbound'" } }, { "pipeline": { "tag": "pipeline_720ed255", - "name": "logs-pfsense.log-1.25.1-haproxy", + "name": "logs-pfsense.log-1.25.2-haproxy", "if": "ctx.event.provider == 'haproxy'" } }, { "pipeline": { "tag": "pipeline_456beba5", - "name": "logs-pfsense.log-1.25.1-php-fpm", + "name": "logs-pfsense.log-1.25.2-php-fpm", "if": "ctx.event.provider == 'php-fpm'" } }, { "pipeline": { "tag": "pipeline_a0d89375", - "name": "logs-pfsense.log-1.25.1-squid", + "name": "logs-pfsense.log-1.25.2-squid", "if": "ctx.event.provider == 'squid'" } }, { "pipeline": { "tag": "pipeline_c2f1ed55", - "name": "logs-pfsense.log-1.25.1-snort", + "name": "logs-pfsense.log-1.25.2-snort", "if": "ctx.event.provider == 'snort'" } }, { "pipeline": { "tag":"pipeline_33db1c9e", - "name": "logs-pfsense.log-1.25.1-suricata", + "name": "logs-pfsense.log-1.25.2-suricata", "if": "ctx.event.provider == 'suricata'" } }, { "drop": { "tag": "drop_9d7c46f8", - "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"snort\", \"suricata\"].contains(ctx.event?.provider)" + "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dnsmasq-dhcp\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"snort\", \"suricata\"].contains(ctx.event?.provider)" } }, { diff --git a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.1-suricata b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2-suricata similarity index 100% rename from salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.1-suricata rename to salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.2-suricata From 5634aed6797d0e34460ade9c4aa099eddc78df1c Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 13 Apr 2026 15:19:39 -0400 Subject: [PATCH 18/24] support minion node descriptions containing spaces --- salt/manager/tools/sbin/so-minion | 4 ++-- setup/so-setup | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 2d5ef448e..76b067817 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -132,8 +132,8 @@ function getinstallinfo() { log "ERROR" "Failed to get install info from $MINION_ID" return 1 fi - - export $(echo "$INSTALLVARS" | xargs) + + while read -r var; do export "$var"; done <<< "$INSTALLVARS" if [ $? -ne 0 ]; then log "ERROR" "Failed to source install variables" return 1 diff --git a/setup/so-setup b/setup/so-setup index 823a379df..1ef88a342 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -219,6 +219,7 @@ if [ -n "$test_profile" ]; then WEBUSER=onionuser@somewhere.invalid WEBPASSWD1=0n10nus3r WEBPASSWD2=0n10nus3r + NODE_DESCRIPTION="${HOSTNAME} - ${install_type}" update_sudoers_for_testing fi From da7c2995b0560689bd8dcd2a2fbc2ad88d5ee138 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 13 Apr 2026 17:09:10 -0400 Subject: [PATCH 19/24] include trailing numbers as an additional test --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 1ef88a342..46b11fc11 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -219,7 +219,7 @@ if [ -n "$test_profile" ]; then WEBUSER=onionuser@somewhere.invalid WEBPASSWD1=0n10nus3r WEBPASSWD2=0n10nus3r - NODE_DESCRIPTION="${HOSTNAME} - ${install_type}" + NODE_DESCRIPTION="${HOSTNAME} - ${install_type} - ${MAINIP}" update_sudoers_for_testing fi From 0405a66c72ec46bb58acc24492f8c142a1c72445 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 13 Apr 2026 16:27:28 -0500 Subject: [PATCH 20/24] enable elastic agent patch release for 9.3.3 --- salt/elasticfleet/defaults.yaml | 1 + .../tools/sbin_jinja/so-elastic-agent-grid-upgrade | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index a3132d3f4..022600083 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -1,5 +1,6 @@ elasticfleet: enabled: False + patch_version: 9.3.3+build202604082258 # Elastic Agent specific patch release. enable_manager_output: True config: server: diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade index 0729531d3..aafc9c368 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade @@ -6,6 +6,11 @@ . /usr/sbin/so-common {%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %} +{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} +{# Optionally override Elasticsearch version for Elastic Agent patch releases #} +{%- if ELASTICFLEETDEFAULTS.elasticfleet.patch_version is defined %} +{%- do ELASTICSEARCHDEFAULTS.update({'elasticsearch': {'version': ELASTICFLEETDEFAULTS.elasticfleet.patch_version}}) %} +{%- endif %} # Only run on Managers if ! is_manager_node; then From d598e20fbbc27fe5155a22d6c838ef20aa692ecc Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 14 Apr 2026 14:55:33 -0500 Subject: [PATCH 21/24] soup 3.1.0 --- salt/manager/tools/sbin/soup | 94 ++++++++++++++++++++++-------------- 1 file changed, 57 insertions(+), 37 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index d25153863..a3b5daa23 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -363,6 +363,7 @@ preupgrade_changes() { echo "Checking to see if changes are needed." [[ "$INSTALLEDVERSION" =~ ^2\.4\.21[0-9]+$ ]] && up_to_3.0.0 + [[ "$INSTALLEDVERSION" == "3.0.0" ]] && up_to_3.1.0 true } @@ -371,6 +372,7 @@ postupgrade_changes() { echo "Running post upgrade processes." [[ "$POSTVERSION" =~ ^2\.4\.21[0-9]+$ ]] && post_to_3.0.0 + [[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0 true } @@ -445,7 +447,6 @@ migrate_pcap_to_suricata() { } up_to_3.0.0() { - determine_elastic_agent_upgrade migrate_pcap_to_suricata INSTALLEDVERSION=3.0.0 @@ -469,6 +470,32 @@ post_to_3.0.0() { ### 3.0.0 End ### +### 3.1.0 Scripts ### + +elasticsearch_backup_index_templates() { + echo "Backing up current elasticsearch index templates in /opt/so/conf/elasticsearch/templates/index/ to /nsm/backup/3.0.0_elasticsearch_index_templates.tar.gz" + tar -czf /nsm/backup/3.0.0_elasticsearch_index_templates.tar.gz -C /opt/so/conf/elasticsearch/templates/index/ . +} + +up_to_3.1.0() { + determine_elastic_agent_upgrade + elasticsearch_backup_index_templates + # Clear existing component template state file. + rm -f /opt/so/state/esfleet_component_templates.json + + + INSTALLEDVERSION=3.1.0 +} + +post_to_3.1.0() { + /usr/sbin/so-kibana-space-defaults + + POSTVERSION=3.1.0 +} + +### 3.1.0 End ### + + repo_sync() { echo "Sync the local repo." su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync." @@ -728,12 +755,12 @@ verify_es_version_compatibility() { local is_active_intermediate_upgrade=1 # supported upgrade paths for SO-ES versions declare -A es_upgrade_map=( - ["8.18.8"]="9.0.8" + ["9.0.8"]="9.3.3" ) # Elasticsearch MUST upgrade through these versions declare -A es_to_so_version=( - ["8.18.8"]="2.4.190-20251024" + ["9.0.8"]="3.0.0-20260331" ) # Get current Elasticsearch version @@ -745,26 +772,17 @@ verify_es_version_compatibility() { exit 160 fi - if ! target_es_version_raw=$(so-yaml.py get $UPDATE_DIR/salt/elasticsearch/defaults.yaml elasticsearch.version); then - # so-yaml.py failed to get the ES version from upgrade versions elasticsearch/defaults.yaml file. Likely they are upgrading to an SO version older than 2.4.110 prior to the ES version pinning and should be OKAY to continue with the upgrade. + if ! target_es_version=$(so-yaml.py get -r $UPDATE_DIR/salt/elasticsearch/defaults.yaml elasticsearch.version); then + echo "Couldn't determine the target Elasticsearch version (post soup version) to ensure compatibility with current Elasticsearch version. Exiting" - # if so-yaml.py failed to get the ES version AND the version we are upgrading to is newer than 2.4.110 then we should bail - if [[ $(cat $UPDATE_DIR/VERSION | cut -d'.' -f3) > 110 ]]; then - echo "Couldn't determine the target Elasticsearch version (post soup version) to ensure compatibility with current Elasticsearch version. Exiting" - - exit 160 - fi - - # allow upgrade to version < 2.4.110 without checking ES version compatibility - return 0 - else - target_es_version=$(sed -n '1p' <<< "$target_es_version_raw") + exit 160 fi for statefile in "${es_required_version_statefile_base}"-*; do [[ -f $statefile ]] || continue - local es_required_version_statefile_value=$(cat "$statefile") + local es_required_version_statefile_value + es_required_version_statefile_value=$(cat "$statefile") if [[ "$es_required_version_statefile_value" == "$target_es_version" ]]; then echo "Intermediate upgrade to ES $target_es_version is in progress. Skipping Elasticsearch version compatibility check." @@ -773,7 +791,7 @@ verify_es_version_compatibility() { fi # use sort to check if es_required_statefile_value is < the current es_version. - if [[ "$(printf '%s\n' $es_required_version_statefile_value $es_version | sort -V | head -n1)" == "$es_required_version_statefile_value" ]]; then + if [[ "$(printf '%s\n' "$es_required_version_statefile_value" "$es_version" | sort -V | head -n1)" == "$es_required_version_statefile_value" ]]; then rm -f "$statefile" continue fi @@ -784,8 +802,7 @@ verify_es_version_compatibility() { echo -e "\n##############################################################################################################################\n" echo "A previously required intermediate Elasticsearch upgrade was detected. Verifying that all Searchnodes/Heavynodes have successfully upgraded Elasticsearch to $es_required_version_statefile_value before proceeding with soup to avoid potential data loss! This command can take up to an hour to complete." - timeout --foreground 4000 bash "$es_verification_script" "$es_required_version_statefile_value" "$statefile" - if [[ $? -ne 0 ]]; then + if ! timeout --foreground 4000 bash "$es_verification_script" "$es_required_version_statefile_value" "$statefile"; then echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n" echo "A previous required intermediate Elasticsearch upgrade to $es_required_version_statefile_value has yet to successfully complete across the grid. Please allow time for all Searchnodes/Heavynodes to have upgraded Elasticsearch to $es_required_version_statefile_value before running soup again to avoid potential data loss!" @@ -802,6 +819,7 @@ verify_es_version_compatibility() { return 0 fi + # shellcheck disable=SC2076 # Do not want a regex here eg usage " 8.18.8 9.0.8 " =~ " 9.0.8 " if [[ " ${es_upgrade_map[$es_version]} " =~ " $target_es_version " || "$es_version" == "$target_es_version" ]]; then # supported upgrade return 0 @@ -810,7 +828,7 @@ verify_es_version_compatibility() { if [[ -z "$compatible_versions" ]]; then # If current ES version is not explicitly defined in the upgrade map, we know they have an intermediate upgrade to do. # We default to the lowest ES version defined in es_to_so_version as $first_es_required_version - local first_es_required_version=$(printf '%s\n' "${!es_to_so_version[@]}" | sort -V | head -n1) + first_es_required_version=$(printf '%s\n' "${!es_to_so_version[@]}" | sort -V | head -n1) next_step_so_version=${es_to_so_version[$first_es_required_version]} required_es_upgrade_version="$first_es_required_version" else @@ -829,7 +847,7 @@ verify_es_version_compatibility() { if [[ $is_airgap -eq 0 ]]; then run_airgap_intermediate_upgrade else - if [[ ! -z $ISOLOC ]]; then + if [[ -n $ISOLOC ]]; then originally_requested_iso_location="$ISOLOC" fi # Make sure ISOLOC is not set. Network installs that used soup -f would have ISOLOC set. @@ -861,7 +879,8 @@ wait_for_salt_minion_with_restart() { } run_airgap_intermediate_upgrade() { - local originally_requested_so_version=$(cat $UPDATE_DIR/VERSION) + local originally_requested_so_version + originally_requested_so_version=$(cat "$UPDATE_DIR/VERSION") # preserve ISOLOC value, so we can try to use it post intermediate upgrade local originally_requested_iso_location="$ISOLOC" @@ -873,7 +892,8 @@ run_airgap_intermediate_upgrade() { while [[ -z "$next_iso_location" ]] || [[ ! -f "$next_iso_location" && ! -b "$next_iso_location" ]]; do # List removable devices if any are present - local removable_devices=$(lsblk -no PATH,SIZE,TYPE,MOUNTPOINTS,RM | awk '$NF==1') + local removable_devices + removable_devices=$(lsblk -no PATH,SIZE,TYPE,MOUNTPOINTS,RM | awk '$NF==1') if [[ -n "$removable_devices" ]]; then echo "PATH SIZE TYPE MOUNTPOINTS RM" echo "$removable_devices" @@ -894,21 +914,21 @@ run_airgap_intermediate_upgrade() { echo "Using $next_iso_location for required intermediary upgrade." exec bash < Date: Tue, 14 Apr 2026 19:26:37 -0500 Subject: [PATCH 22/24] check for addon-index templates dir before attempting to load addon index templates --- salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load index 9ad0f418a..840639a32 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load @@ -207,7 +207,7 @@ else fi # Start loading addon templates -if [[ (-f "$SO_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" && ! -f "$ADDON_STATEFILE_SUCCESS") || ("$IS_HEAVYNODE" == "false" && "$FORCE" == "true") ]]; then +if [[ (-d "$ADDON_TEMPLATES_DIR" && -f "$SO_STATEFILE_SUCCESS" && "$IS_HEAVYNODE" == "false" && ! -f "$ADDON_STATEFILE_SUCCESS") || (-d "$ADDON_TEMPLATES_DIR" && "$IS_HEAVYNODE" == "false" && "$FORCE" == "true") ]]; then check_elasticsearch_responsive From 88582c94e8943abe301408a4bf503f070747e1f0 Mon Sep 17 00:00:00 2001 From: Jorge Reyes <94730068+reyesj2@users.noreply.github.com> Date: Wed, 15 Apr 2026 15:04:20 -0500 Subject: [PATCH 23/24] remove foxtrot version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 0e0d1ae9a..fd2a01863 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.0.0-foxtrot \ No newline at end of file +3.1.0 From ba00ae8a7b9b27cea2372092c3903e0391858805 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 16 Apr 2026 14:41:25 -0500 Subject: [PATCH 24/24] supress noisy warning from ES 9.3.3 --- salt/elasticsearch/files/log4j2.properties | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/elasticsearch/files/log4j2.properties b/salt/elasticsearch/files/log4j2.properties index b29378d6a..050071581 100644 --- a/salt/elasticsearch/files/log4j2.properties +++ b/salt/elasticsearch/files/log4j2.properties @@ -45,3 +45,7 @@ appender.rolling_json.strategy.action.condition.nested_condition.age = 1D rootLogger.level = info rootLogger.appenderRef.rolling.ref = rolling rootLogger.appenderRef.rolling_json.ref = rolling_json + +# Suppress NotEntitledException WARNs (ES 9.3.3 bug) +logger.entitlement_security.name = org.elasticsearch.entitlement.runtime.policy.PolicyManager.x-pack-security.org.elasticsearch.security.org.elasticsearch.xpack.security +logger.entitlement_security.level = error \ No newline at end of file