From 8101bc4941c230418aeb3b8a675739392b3a0139 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 6 Apr 2026 15:08:30 -0500 Subject: [PATCH] ES 9.3.2 --- .../elastic-defend-endpoints.json | 2 +- .../grid-nodes_general/import-evtx-logs.json | 2 +- salt/elasticsearch/defaults.yaml | 2 +- ...nse.log-1.23.1 => logs-pfsense.log-1.25.1} | 83 ++++++++++++++++--- ...icata => logs-pfsense.log-1.25.1-suricata} | 0 .../tools/sbin_jinja/so-kibana-space-defaults | 2 +- 6 files changed, 76 insertions(+), 15 deletions(-) rename salt/elasticsearch/files/ingest/{logs-pfsense.log-1.23.1 => logs-pfsense.log-1.25.1} (74%) rename salt/elasticsearch/files/ingest/{logs-pfsense.log-1.23.1-suricata => logs-pfsense.log-1.25.1-suricata} (100%) diff --git a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json index debfc73a3..c27da26f7 100644 --- a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json +++ b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json @@ -5,7 +5,7 @@ "package": { "name": "endpoint", "title": "Elastic Defend", - "version": "9.0.2", + "version": "9.3.0", "requires_root": true }, "enabled": true, diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json index 50ffd5dc7..3066303d9 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json @@ -23,7 +23,7 @@ "\\.gz$" ], "include_files": [], - "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.6.1\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.1.2\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.6.1\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.6.1\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.1.2\n- add_fields:\n target: data_stream\n fields:\n dataset: import", + "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.13.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.6.0\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.13.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.13.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.6.0\n- add_fields:\n target: data_stream\n fields:\n dataset: import", "tags": [ "import" ], diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index d0ab0f959..f355601dc 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1,6 +1,6 @@ elasticsearch: enabled: false - version: 9.0.8 + version: 9.3.2 index_clean: true vm: max_map_count: 1048576 diff --git a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.1 b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.1 similarity index 74% rename from salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.1 rename to salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.1 index d3354f363..3037ce77a 100644 --- a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.1 +++ b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.1 @@ -10,24 +10,28 @@ "processors": [ { "set": { + "tag": "set_ecs_version_f5923549", "field": "ecs.version", "value": "8.17.0" } }, { "set": { + "tag": "set_observer_vendor_ad9d35cc", "field": "observer.vendor", "value": "netgate" } }, { "set": { + "tag": "set_observer_type_5dddf3ba", "field": "observer.type", "value": "firewall" } }, { "rename": { + "tag": "rename_message_to_event_original_56a77271", "field": "message", "target_field": "event.original", "ignore_missing": true, @@ -36,12 +40,14 @@ }, { "set": { + "tag": "set_event_kind_de80643c", "field": "event.kind", "value": "event" } }, { "set": { + "tag": "set_event_timezone_4ca44cac", "field": "event.timezone", "value": "{{{_tmp.tz_offset}}}", "if": "ctx._tmp?.tz_offset != null && ctx._tmp?.tz_offset != 'local'" @@ -49,6 +55,7 @@ }, { "grok": { + "tag": "grok_event_original_27d9c8c7", "description": "Parse syslog header", "field": "event.original", "patterns": [ @@ -72,6 +79,7 @@ }, { "date": { + "tag": "date__tmp_timestamp8601_to_timestamp_6ac9d3ce", "if": "ctx._tmp.timestamp8601 != null", "field": "_tmp.timestamp8601", "target_field": "@timestamp", @@ -82,6 +90,7 @@ }, { "date": { + "tag": "date__tmp_timestamp_to_timestamp_f21e536e", "if": "ctx.event?.timezone != null && ctx._tmp?.timestamp != null", "field": "_tmp.timestamp", "target_field": "@timestamp", @@ -95,6 +104,7 @@ }, { "grok": { + "tag": "grok_process_name_cef3d489", "description": "Set Event Provider", "field": "process.name", "patterns": [ @@ -107,71 +117,83 @@ }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-firewall", + "tag": "pipeline_e16851a7", + "name": "logs-pfsense.log-1.25.1-firewall", "if": "ctx.event.provider == 'filterlog'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-openvpn", + "tag": "pipeline_828590b5", + "name": "logs-pfsense.log-1.25.1-openvpn", "if": "ctx.event.provider == 'openvpn'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-ipsec", + "tag": "pipeline_9d37039c", + "name": "logs-pfsense.log-1.25.1-ipsec", "if": "ctx.event.provider == 'charon'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-dhcp", + "tag": "pipeline_ad56bbca", + "name": "logs-pfsense.log-1.25.1-dhcp", "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\"].contains(ctx.event.provider)" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-unbound", + "tag": "pipeline_dd85553d", + "name": "logs-pfsense.log-1.25.1-unbound", "if": "ctx.event.provider == 'unbound'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-haproxy", + "tag": "pipeline_720ed255", + "name": "logs-pfsense.log-1.25.1-haproxy", "if": "ctx.event.provider == 'haproxy'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-php-fpm", + "tag": "pipeline_456beba5", + "name": "logs-pfsense.log-1.25.1-php-fpm", "if": "ctx.event.provider == 'php-fpm'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-squid", + "tag": "pipeline_a0d89375", + "name": "logs-pfsense.log-1.25.1-squid", "if": "ctx.event.provider == 'squid'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-snort", + "tag": "pipeline_c2f1ed55", + "name": "logs-pfsense.log-1.25.1-snort", "if": "ctx.event.provider == 'snort'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.23.1-suricata", + "tag":"pipeline_33db1c9e", + "name": "logs-pfsense.log-1.25.1-suricata", "if": "ctx.event.provider == 'suricata'" } }, { "drop": { + "tag": "drop_9d7c46f8", "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"snort\", \"suricata\"].contains(ctx.event?.provider)" } }, { "append": { + "tag": "append_event_category_4780a983", "field": "event.category", "value": "network", "if": "ctx.network != null" @@ -179,6 +201,7 @@ }, { "convert": { + "tag": "convert_source_address_to_source_ip_f5632a20", "field": "source.address", "target_field": "source.ip", "type": "ip", @@ -188,6 +211,7 @@ }, { "convert": { + "tag": "convert_destination_address_to_destination_ip_f1388f0c", "field": "destination.address", "target_field": "destination.ip", "type": "ip", @@ -197,6 +221,7 @@ }, { "set": { + "tag": "set_network_type_1f1d940a", "field": "network.type", "value": "ipv6", "if": "ctx.source?.ip != null && ctx.source.ip.contains(\":\")" @@ -204,6 +229,7 @@ }, { "set": { + "tag": "set_network_type_69deca38", "field": "network.type", "value": "ipv4", "if": "ctx.source?.ip != null && ctx.source.ip.contains(\".\")" @@ -211,6 +237,7 @@ }, { "geoip": { + "tag": "geoip_source_ip_to_source_geo_da2e41b2", "field": "source.ip", "target_field": "source.geo", "ignore_missing": true @@ -218,6 +245,7 @@ }, { "geoip": { + "tag": "geoip_destination_ip_to_destination_geo_ab5e2968", "field": "destination.ip", "target_field": "destination.geo", "ignore_missing": true @@ -225,6 +253,7 @@ }, { "geoip": { + "tag": "geoip_source_ip_to_source_as_28d69883", "ignore_missing": true, "database_file": "GeoLite2-ASN.mmdb", "field": "source.ip", @@ -237,6 +266,7 @@ }, { "geoip": { + "tag": "geoip_destination_ip_to_destination_as_8a007787", "database_file": "GeoLite2-ASN.mmdb", "field": "destination.ip", "target_field": "destination.as", @@ -249,6 +279,7 @@ }, { "rename": { + "tag": "rename_source_as_asn_to_source_as_number_a917047d", "field": "source.as.asn", "target_field": "source.as.number", "ignore_missing": true @@ -256,6 +287,7 @@ }, { "rename": { + "tag": "rename_source_as_organization_name_to_source_as_organization_name_f1362d0b", "field": "source.as.organization_name", "target_field": "source.as.organization.name", "ignore_missing": true @@ -263,6 +295,7 @@ }, { "rename": { + "tag": "rename_destination_as_asn_to_destination_as_number_3b459fcd", "field": "destination.as.asn", "target_field": "destination.as.number", "ignore_missing": true @@ -270,6 +303,7 @@ }, { "rename": { + "tag": "rename_destination_as_organization_name_to_destination_as_organization_name_814bd459", "field": "destination.as.organization_name", "target_field": "destination.as.organization.name", "ignore_missing": true @@ -277,12 +311,14 @@ }, { "community_id": { + "tag": "community_id_d2308e7a", "target_field": "network.community_id", "ignore_failure": true } }, { "grok": { + "tag": "grok_observer_ingress_interface_name_968018d3", "field": "observer.ingress.interface.name", "patterns": [ "%{DATA}.%{NONNEGINT:observer.ingress.vlan.id}" @@ -293,6 +329,7 @@ }, { "set": { + "tag": "set_network_vlan_id_efd4d96a", "field": "network.vlan.id", "copy_from": "observer.ingress.vlan.id", "ignore_empty_value": true @@ -300,6 +337,7 @@ }, { "append": { + "tag": "append_related_ip_c1a6356b", "field": "related.ip", "value": "{{{destination.ip}}}", "allow_duplicates": false, @@ -308,6 +346,7 @@ }, { "append": { + "tag": "append_related_ip_8121c591", "field": "related.ip", "value": "{{{source.ip}}}", "allow_duplicates": false, @@ -316,6 +355,7 @@ }, { "append": { + "tag": "append_related_ip_53b62ed8", "field": "related.ip", "value": "{{{source.nat.ip}}}", "allow_duplicates": false, @@ -324,6 +364,7 @@ }, { "append": { + "tag": "append_related_hosts_6f162628", "field": "related.hosts", "value": "{{{destination.domain}}}", "if": "ctx.destination?.domain != null" @@ -331,6 +372,7 @@ }, { "append": { + "tag": "append_related_user_c036eec2", "field": "related.user", "value": "{{{user.name}}}", "if": "ctx.user?.name != null" @@ -338,6 +380,7 @@ }, { "set": { + "tag": "set_network_direction_cb1e3125", "field": "network.direction", "value": "{{{network.direction}}}bound", "if": "ctx.network?.direction != null && ctx.network?.direction =~ /^(in|out)$/" @@ -345,6 +388,7 @@ }, { "remove": { + "tag": "remove_a82e20f2", "field": [ "_tmp" ], @@ -353,11 +397,21 @@ }, { "script": { + "tag": "script_a7f2c062", "lang": "painless", "description": "This script processor iterates over the whole document to remove fields with null values.", "source": "void handleMap(Map map) {\n for (def x : map.values()) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n map.values().removeIf(v -> v == null || (v instanceof String && v == \"-\"));\n}\nvoid handleList(List list) {\n for (def x : list) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n}\nhandleMap(ctx);\n" } }, + { + "append": { + "tag": "append_preserve_original_event_on_error", + "field": "tags", + "value": "preserve_original_event", + "allow_duplicates": false, + "if": "ctx.error?.message != null" + } + }, { "pipeline": { "name": "global@custom", @@ -405,7 +459,14 @@ { "append": { "field": "error.message", - "value": "{{{ _ingest.on_failure_message }}}" + "value": "Processor '{{{ _ingest.on_failure_processor_type }}}' {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}'" + } + }, + { + "append": { + "field": "tags", + "value": "preserve_original_event", + "allow_duplicates": false } } ] diff --git a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.1-suricata b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.1-suricata similarity index 100% rename from salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.1-suricata rename to salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.1-suricata diff --git a/salt/kibana/tools/sbin_jinja/so-kibana-space-defaults b/salt/kibana/tools/sbin_jinja/so-kibana-space-defaults index fcb80e606..d0447f514 100755 --- a/salt/kibana/tools/sbin_jinja/so-kibana-space-defaults +++ b/salt/kibana/tools/sbin_jinja/so-kibana-space-defaults @@ -9,5 +9,5 @@ SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http: # Disable certain Features from showing up in the Kibana UI echo echo "Setting up default Kibana Space:" -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","entityManager"]} ' >> /opt/so/log/kibana/misc.log +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","searchQueryRules","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","securitySolutionRulesV1","entityManager","streams","cloudConnect","slo"]} ' >> /opt/so/log/kibana/misc.log echo