Merge pull request #14825 from Security-Onion-Solutions/foxtrot

ES 8.18.3

salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json

@@ -5,18 +5,20 @@
   "package": {
     "name": "endpoint",
     "title": "Elastic Defend",
-	  "version": "8.17.0",
+    "version": "8.18.1",
     "requires_root": true
   },
   "enabled": true,
-	"policy_id": "endpoints-initial",
+  "policy_ids": [
+    "endpoints-initial"
+  ],
   "vars": {},
   "inputs": [
     {
-		"type": "endpoint",
+      "type": "ENDPOINT_INTEGRATION_CONFIG",
       "enabled": true,
       "config": {
-		  "integration_config": {
+        "_config": {
           "value": {
             "type": "endpoint",
             "endpointConfig": {
@@ -29,4 +31,3 @@
     }
   ]
 }
-  

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -20,7 +20,7 @@
             ],
             "data_stream.dataset": "import",
             "custom": "",
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-1.67.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-2.5.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-1.67.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-1.67.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-2.5.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.3.1\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.0.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.3.1\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.3.1\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.0.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
             "tags": [
               "import"
             ]

salt/elasticfleet/integration-defaults.map.jinja

@@ -4,6 +4,7 @@
 
 
 {% import_json '/opt/so/state/esfleet_package_components.json' as ADDON_PACKAGE_COMPONENTS %}
+{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %}
 {% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
 
 {% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
@@ -45,7 +46,10 @@
     'synthetics.browser_screenshot': 'synthetics-browser.screenshot',
     'synthetics.http': 'synthetics-http',
     'synthetics.icmp': 'synthetics-icmp',
-    'synthetics.tcp': 'synthetics-tcp'
+    'synthetics.tcp': 'synthetics-tcp',
+    'swimlane.swimlane_api': 'swimlane.api',
+    'swimlane.tenant_api': 'swimlane.tenant',
+    'swimlane.turbine_api': 'turbine.api'
    } %}
 
 {% for pkg in ADDON_PACKAGE_COMPONENTS %}
@@ -63,31 +67,50 @@
 {%           set integration_type = "" %}
 {%         endif %}
 {%         set component_name = pkg.name ~ "." ~ pattern.title %}
+{%         set index_pattern = pattern.name %}
+
 {#         fix weirdly named components #}
 {%         if component_name in WEIRD_INTEGRATIONS %}
 {%           set component_name = WEIRD_INTEGRATIONS[component_name] %}
 {%         endif %}
+
+{#         create duplicate of component_name, so we can split generics from @custom component templates in the index template below and overwrite the default @package when needed
+           eg. having to replace unifiedlogs.generic@package with filestream.generic@package, but keep the ability to customize unifiedlogs.generic@custom and its ILM policy #}
+{%         set custom_component_name = component_name %}
+
+{#         duplicate integration_type to assist with sometimes needing to overwrite component templates with 'logs-filestream.generic@package' (there is no metrics-filestream.generic@package) #}
+{%         set generic_integration_type = integration_type %}
+
 {#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
 {%           set component_name_x = component_name.replace(".","_x_") %}
 {#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
 {%           set integration_key = "so-" ~ integration_type ~ component_name_x %}
 
+{#         if its a .generic template make sure that a .generic@package for the integration exists. Else default to logs-filestream.generic@package #}
+{%         if ".generic" in component_name and integration_type ~ component_name ~ "@package" not in INSTALLED_COMPONENT_TEMPLATES %}
+{#           these generic templates by default are directed to index_pattern of 'logs-generic-*', overwrite that here to point to eg gcp_pubsub.generic-* #}
+{%           set index_pattern = integration_type ~ component_name ~ "-*" %}
+{#           includes use of .generic component template, but it doesn't exist in installed component templates. Redirect it to filestream.generic@package #}
+{%           set component_name = "filestream.generic" %}
+{%           set generic_integration_type = "logs-" %}
+{%         endif %}
+
 {#           Default integration settings #}
 {%           set integration_defaults = {
                "index_sorting": false,
                "index_template": {
-                  "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
+                 "composed_of": [generic_integration_type ~ component_name ~ "@package", integration_type ~ custom_component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
                  "data_stream": {
                    "allow_custom_routing": false,
                    "hidden": false
                  },
-                  "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
+                 "ignore_missing_component_templates": [integration_type ~ custom_component_name ~ "@custom"],
-                  "index_patterns": [pattern.name],
+                 "index_patterns": [index_pattern],
                  "priority": 501,
                  "template": {
                    "settings": {
                      "index": {
-                              "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
+                       "lifecycle": {"name": "so-" ~ integration_type ~ custom_component_name ~ "-logs"},
                          "number_of_replicas": 0
                      }
                    }
@@ -126,6 +149,7 @@
                  }
                }
              } %}
+
 {%         do ADDON_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
 {%       endfor %}
 {%     endif %}

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load

@@ -19,6 +19,7 @@ BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json
 BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json
 BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json
 PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
+COMPONENT_TEMPLATES=/opt/so/state/esfleet_component_templates.json
 
 PENDING_UPDATE=false
 
@@ -155,6 +156,11 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
         # Write out file for generating index/component/ilm templates
         latest_installed_package_list=$(elastic_fleet_installed_packages)
         echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS
+        if retry 3 1 "so-elasticsearch-query / --fail --output /dev/null"; then
+            # Refresh installed component template list
+            latest_component_templates_list=$(so-elasticsearch-query _component_template | jq '.component_templates[] | .name' | jq -s '.')
+            echo $latest_component_templates_list > $COMPONENT_TEMPLATES
+        fi
 
     else
         # This is the installation of add-on integrations and upgrade of existing integrations. Exiting without error, next highstate will attempt to re-run.

salt/elasticsearch/defaults.yaml

@@ -1,6 +1,6 @@
 elasticsearch:
   enabled: false
-  version: 8.17.3
+  version: 8.18.3
   index_clean: true
   config:
     action:

salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.0

@@ -107,61 +107,61 @@
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-firewall",
+        "name": "logs-pfsense.log-1.23.0-firewall",
         "if": "ctx.event.provider == 'filterlog'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-openvpn",
+        "name": "logs-pfsense.log-1.23.0-openvpn",
         "if": "ctx.event.provider == 'openvpn'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-ipsec",
+        "name": "logs-pfsense.log-1.23.0-ipsec",
         "if": "ctx.event.provider == 'charon'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-dhcp",
+        "name": "logs-pfsense.log-1.23.0-dhcp",
         "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\"].contains(ctx.event.provider)"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-unbound",
+        "name": "logs-pfsense.log-1.23.0-unbound",
         "if": "ctx.event.provider == 'unbound'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-haproxy",
+        "name": "logs-pfsense.log-1.23.0-haproxy",
         "if": "ctx.event.provider == 'haproxy'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-php-fpm",
+        "name": "logs-pfsense.log-1.23.0-php-fpm",
         "if": "ctx.event.provider == 'php-fpm'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-squid",
+        "name": "logs-pfsense.log-1.23.0-squid",
         "if": "ctx.event.provider == 'squid'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-snort",
+        "name": "logs-pfsense.log-1.23.0-snort",
         "if": "ctx.event.provider == 'snort'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.21.0-suricata",
+        "name": "logs-pfsense.log-1.23.0-suricata",
         "if": "ctx.event.provider == 'suricata'"
       }
     },
@@ -358,14 +358,6 @@
         "source": "void handleMap(Map map) {\n  for (def x : map.values()) {\n    if (x instanceof Map) {\n        handleMap(x);\n    } else if (x instanceof List) {\n        handleList(x);\n    }\n  }\n  map.values().removeIf(v -> v == null || (v instanceof String && v == \"-\"));\n}\nvoid handleList(List list) {\n  for (def x : list) {\n      if (x instanceof Map) {\n          handleMap(x);\n      } else if (x instanceof List) {\n          handleList(x);\n      }\n  }\n}\nhandleMap(ctx);\n"
       }
     },
-    {
-      "remove": {
-        "field": "event.original",
-        "if": "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))",
-        "ignore_failure": true,
-        "ignore_missing": true
-      }
-    },
     {
       "pipeline": {
         "name": "global@custom",

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load

@@ -136,7 +136,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then
       TEMPLATE=${i::-14}
       COMPONENT_PATTERN=${TEMPLATE:3}
       MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery")
-      if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ logs-http_endpoint\.generic|logs-winlog\.winlog ]]; then
+      if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ \.generic|logs-winlog\.winlog ]]; then
         load_failures=$((load_failures+1))
         echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures"
       else

salt/kibana/defaults.yaml

@@ -22,7 +22,7 @@ kibana:
           - default
           - file
     migrations:
-      discardCorruptObjects: "8.17.3"
+      discardCorruptObjects: "8.18.3"
     telemetry:
       enabled: False
     security:

salt/kibana/tools/sbin_jinja/so-kibana-space-defaults

@@ -13,6 +13,6 @@ echo "Setting up default Space:"
 {% if HIGHLANDER %}
 curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["enterpriseSearch"]} ' >> /opt/so/log/kibana/misc.log
 {% else %}
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV2","inventory","dataQuality","actions"]} ' >> /opt/so/log/kibana/misc.log
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCasesV3","inventory","dataQuality","searchSynonyms","enterpriseSearchApplications","enterpriseSearchAnalytics","securitySolutionTimeline","securitySolutionNotes","entityManager"]} ' >> /opt/so/log/kibana/misc.log
 {% endif %}
 echo

salt/manager/managed_soc_annotations.sls

@@ -5,7 +5,7 @@
 
 {# Managed elasticsearch/soc_elasticsearch.yaml file for adding integration configuration items to UI #}
 {% set managed_integrations = salt['pillar.get']('elasticsearch:managed_integrations', []) %}
-{% if managed_integrations %}
+{% if managed_integrations and salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') %}
 {%   from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %}
 {%   set addon_integration_keys = ADDON_INTEGRATION_DEFAULTS.keys() %}
 {%   set matched_integration_names = [] %}

salt/manager/tools/sbin/soup

@@ -590,6 +590,12 @@ post_to_2.4.160() {
 post_to_2.4.170() {
   echo "Regenerating Elastic Agent Installers"
   /sbin/so-elastic-agent-gen-installers
+
+  # Update kibana default space
+  salt-call state.apply kibana.config queue=True
+  echo "Updating Kibana default space"
+  /usr/sbin/so-kibana-space-defaults
+
   POSTVERSION=2.4.170
 }
 
@@ -802,9 +808,6 @@ up_to_2.4.130() {
   # Remove any old Elastic Defend config files
   rm -f /opt/so/conf/elastic-fleet/integrations/endpoints-initial/elastic-defend-endpoints.json
 
-  # Elastic Update for this release, so download Elastic Agent files
-  determine_elastic_agent_upgrade
-
   # Ensure override exists to allow nmcli access to other devices
   touch /etc/NetworkManager/conf.d/10-globally-managed-devices.conf
 
@@ -847,6 +850,9 @@ up_to_2.4.170() {
     touch /opt/so/saltstack/local/pillar/$state/adv_$state.sls /opt/so/saltstack/local/pillar/$state/soc_$state.sls
   done
 
+  # Elastic Update for this release, so download Elastic Agent files
+  determine_elastic_agent_upgrade
+
   INSTALLEDVERSION=2.4.170
 }