diff --git a/VERSION b/VERSION index 452820224..c3344819a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.0-foxtrot \ No newline at end of file +2.4.170 \ No newline at end of file diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json index 46717f3e1..415e333e7 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json @@ -20,7 +20,7 @@ ], "data_stream.dataset": "import", "custom": "", - "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-1.67.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-2.5.0\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-1.67.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-1.67.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-2.5.0\n- add_fields:\n target: data_stream\n fields:\n dataset: import", + "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.3.1\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.0.0\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.3.1\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.3.1\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.0.0\n- add_fields:\n target: data_stream\n fields:\n dataset: import", "tags": [ "import" ] diff --git a/salt/elasticfleet/integration-defaults.map.jinja b/salt/elasticfleet/integration-defaults.map.jinja index 5449e791e..62e37b229 100644 --- a/salt/elasticfleet/integration-defaults.map.jinja +++ b/salt/elasticfleet/integration-defaults.map.jinja @@ -4,6 +4,7 @@ {% import_json '/opt/so/state/esfleet_package_components.json' as ADDON_PACKAGE_COMPONENTS %} +{% import_json '/opt/so/state/esfleet_package_components.json' as INSTALLED_COMPONENT_TEMPLATES %} {% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} {% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %} @@ -62,70 +63,89 @@ {% else %} {% set integration_type = "" %} {% endif %} -{% set component_name = pkg.name ~ "." ~ pattern.title %} -{# fix weirdly named components #} -{% if component_name in WEIRD_INTEGRATIONS %} -{% set component_name = WEIRD_INTEGRATIONS[component_name] %} -{% endif %} +{% set component_name = pkg.name ~ "." ~ pattern.title %} +{% set index_pattern = pattern.name %} + +{# fix weirdly named components #} +{% if component_name in WEIRD_INTEGRATIONS %} +{% set component_name = WEIRD_INTEGRATIONS[component_name] %} +{% endif %} + +{# create duplicate of component_name, so we can split generics from @custom component templates in the index template below and overwrite the default @package when needed + eg. having to replace unifiedlogs.generic@package with filestream.generic@package, but keep the ability to customize unifiedlogs.generic@custom and its ILM policy #} +{% set custom_component_name = component_name %} + +{# duplicate integration_type to assist with sometimes needing to overwrite component templates with 'logs-filestream.generic@package' (there is no metrics-filestream.generic@package) #} +{% set generic_integration_type = integration_type %} + {# component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #} {% set component_name_x = component_name.replace(".","_x_") %} {# pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #} {% set integration_key = "so-" ~ integration_type ~ component_name_x %} +{% if ".generic" in component_name and component_name not in INSTALLED_COMPONENT_TEMPLATES %} +{# these generic templates by default are directed to index_pattern of 'logs-generic-*', overwrite that here to point to eg gcp_pubsub.generic-* #} +{% set index_pattern = integration_type ~ component_name ~ "-*" %} +{# includes use of .generic component template, but it doesn't exist in installed component templates. Redirect it to filestream.generic@package #} +{% set component_name = "filestream.generic" %} +{% set generic_integration_type = "logs-" %} +{% endif %} + {# Default integration settings #} {% set integration_defaults = { - "index_sorting": false, - "index_template": { - "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"], - "data_stream": { - "allow_custom_routing": false, - "hidden": false - }, - "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"], - "index_patterns": [pattern.name], - "priority": 501, - "template": { - "settings": { - "index": { - "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"}, - "number_of_replicas": 0 - } - } - } - }, - "policy": { - "phases": { - "cold": { - "actions": { - "set_priority": {"priority": 0} - }, - "min_age": "60d" + "index_sorting": false, + "index_template": { + "composed_of": [generic_integration_type ~ component_name ~ "@package", integration_type ~ custom_component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"], + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "ignore_missing_component_templates": [integration_type ~ custom_component_name ~ "@custom"], + "index_patterns": [index_pattern], + "priority": 501, + "template": { + "settings": { + "index": { + "lifecycle": {"name": "so-" ~ integration_type ~ custom_component_name ~ "-logs"}, + "number_of_replicas": 0 + } + } + } + }, + "policy": { + "phases": { + "cold": { + "actions": { + "set_priority": {"priority": 0} + }, + "min_age": "60d" + }, + "delete": { + "actions": { + "delete": {} + }, + "min_age": "365d" + }, + "hot": { + "actions": { + "rollover": { + "max_age": "30d", + "max_primary_shard_size": "50gb" + }, + "set_priority": {"priority": 100} }, - "delete": { - "actions": { - "delete": {} - }, - "min_age": "365d" - }, - "hot": { - "actions": { - "rollover": { - "max_age": "30d", - "max_primary_shard_size": "50gb" - }, - "set_priority": {"priority": 100} - }, - "min_age": "0ms" - }, - "warm": { - "actions": { - "set_priority": {"priority": 50} - }, - "min_age": "30d" - } - } - } - } %} + "min_age": "0ms" + }, + "warm": { + "actions": { + "set_priority": {"priority": 50} + }, + "min_age": "30d" + } + } + } + } %} + {% do ADDON_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %} {% endfor %} {% endif %} diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load index 26d775e82..df01608f5 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load @@ -19,6 +19,7 @@ BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json +COMPONENT_TEMPLATES=/opt/so/state/esfleet_component_templates.txt PENDING_UPDATE=false @@ -155,6 +156,9 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then # Write out file for generating index/component/ilm templates latest_installed_package_list=$(elastic_fleet_installed_packages) echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS + # Refresh installed component template list + latest_component_templates_list=$(so-elasticsearch-query _component_template | jq '.component_templates[] | .name' | jq -s '.') + echo $latest_component_templates_list > $COMPONENT_TEMPLATES else # This is the installation of add-on integrations and upgrade of existing integrations. Exiting without error, next highstate will attempt to re-run. diff --git a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.21.0 b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.0 similarity index 93% rename from salt/elasticsearch/files/ingest/logs-pfsense.log-1.21.0 rename to salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.0 index 7c4f2575f..e79b91b26 100644 --- a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.21.0 +++ b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.0 @@ -107,61 +107,61 @@ }, { "pipeline": { - "name": "logs-pfsense.log-1.21.0-firewall", + "name": "logs-pfsense.log-1.23.0-firewall", "if": "ctx.event.provider == 'filterlog'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.21.0-openvpn", + "name": "logs-pfsense.log-1.23.0-openvpn", "if": "ctx.event.provider == 'openvpn'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.21.0-ipsec", + "name": "logs-pfsense.log-1.23.0-ipsec", "if": "ctx.event.provider == 'charon'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.21.0-dhcp", + "name": "logs-pfsense.log-1.23.0-dhcp", "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\"].contains(ctx.event.provider)" } }, { "pipeline": { - "name": "logs-pfsense.log-1.21.0-unbound", + "name": "logs-pfsense.log-1.23.0-unbound", "if": "ctx.event.provider == 'unbound'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.21.0-haproxy", + "name": "logs-pfsense.log-1.23.0-haproxy", "if": "ctx.event.provider == 'haproxy'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.21.0-php-fpm", + "name": "logs-pfsense.log-1.23.0-php-fpm", "if": "ctx.event.provider == 'php-fpm'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.21.0-squid", + "name": "logs-pfsense.log-1.23.0-squid", "if": "ctx.event.provider == 'squid'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.21.0-snort", + "name": "logs-pfsense.log-1.23.0-snort", "if": "ctx.event.provider == 'snort'" } }, { "pipeline": { - "name": "logs-pfsense.log-1.21.0-suricata", + "name": "logs-pfsense.log-1.23.0-suricata", "if": "ctx.event.provider == 'suricata'" } }, @@ -358,14 +358,6 @@ "source": "void handleMap(Map map) {\n for (def x : map.values()) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n map.values().removeIf(v -> v == null || (v instanceof String && v == \"-\"));\n}\nvoid handleList(List list) {\n for (def x : list) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n}\nhandleMap(ctx);\n" } }, - { - "remove": { - "field": "event.original", - "if": "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))", - "ignore_failure": true, - "ignore_missing": true - } - }, { "pipeline": { "name": "global@custom", diff --git a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.21.0-suricata b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.0-suricata similarity index 100% rename from salt/elasticsearch/files/ingest/logs-pfsense.log-1.21.0-suricata rename to salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.0-suricata diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load index 758f601eb..ba5767255 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -136,7 +136,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then TEMPLATE=${i::-14} COMPONENT_PATTERN=${TEMPLATE:3} MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery") - if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ logs-http_endpoint\.generic|logs-winlog\.winlog ]]; then + if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ *\.generic|logs-winlog\.winlog ]]; then load_failures=$((load_failures+1)) echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures" else diff --git a/salt/manager/managed_soc_annotations.sls b/salt/manager/managed_soc_annotations.sls index 17621f973..9ef0b1ea1 100644 --- a/salt/manager/managed_soc_annotations.sls +++ b/salt/manager/managed_soc_annotations.sls @@ -5,7 +5,7 @@ {# Managed elasticsearch/soc_elasticsearch.yaml file for adding integration configuration items to UI #} {% set managed_integrations = salt['pillar.get']('elasticsearch:managed_integrations', []) %} -{% if managed_integrations %} +{% if managed_integrations and salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') %} {% from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %} {% set addon_integration_keys = ADDON_INTEGRATION_DEFAULTS.keys() %} {% set matched_integration_names = [] %} diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index f361da688..0c6ebfd26 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -802,9 +802,6 @@ up_to_2.4.130() { # Remove any old Elastic Defend config files rm -f /opt/so/conf/elastic-fleet/integrations/endpoints-initial/elastic-defend-endpoints.json - # Elastic Update for this release, so download Elastic Agent files - determine_elastic_agent_upgrade - # Ensure override exists to allow nmcli access to other devices touch /etc/NetworkManager/conf.d/10-globally-managed-devices.conf @@ -847,6 +844,9 @@ up_to_2.4.170() { touch /opt/so/saltstack/local/pillar/$state/adv_$state.sls /opt/so/saltstack/local/pillar/$state/soc_$state.sls done + # Elastic Update for this release, so download Elastic Agent files + determine_elastic_agent_upgrade + INSTALLEDVERSION=2.4.170 }