From 216869859563f3025a8313f2f7f0e416ec7b79e0 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 22 Jan 2024 20:27:19 -0500 Subject: [PATCH 01/38] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9cf89c6c7..7d52aac7f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.50 +2.4.0-foxtrot From 81d874c6ae48e416715bdda6dd0e8b35c5cd24a2 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 27 Jun 2024 10:42:58 -0400 Subject: [PATCH 02/38] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 38f621b25..7d52aac7f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.90 +2.4.0-foxtrot From 5743189eef080d2bb6462aebc674672b86684420 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 27 Jun 2024 10:47:46 -0400 Subject: [PATCH 03/38] Elastic 8.14.1 --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 05c47a6c1..02d16ad2c 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -8,7 +8,7 @@ # Elastic agent is not managed by salt. Because of this we must store this base information in a # script that accompanies the soup system. Since so-common is one of those special soup files, # and since this same logic is required during installation, it's included in this file. -ELASTIC_AGENT_TARBALL_VERSION="8.10.4" +ELASTIC_AGENT_TARBALL_VERSION="8.14.1" ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5" ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" From 847638442bd181725ee621ab148805e5130eff09 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 27 Jun 2024 10:48:28 -0400 Subject: [PATCH 04/38] Elastic 8.14.1 --- .../integrations/elastic-defend/elastic-defend-endpoints.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json index de35f803b..15f08a151 100644 --- a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json +++ b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json @@ -5,7 +5,7 @@ "package": { "name": "endpoint", "title": "Elastic Defend", - "version": "8.10.2" + "version": "8.14.0" }, "enabled": true, "policy_id": "endpoints-initial", From 8eb6dcc5b7df7f15bb62ce5d57ae13b695e2cf57 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 27 Jun 2024 10:49:06 -0400 Subject: [PATCH 05/38] Elastic 8.14.1 --- salt/kibana/files/config_saved_objects.ndjson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/files/config_saved_objects.ndjson b/salt/kibana/files/config_saved_objects.ndjson index bc503debb..f3370d053 100644 --- a/salt/kibana/files/config_saved_objects.ndjson +++ b/salt/kibana/files/config_saved_objects.ndjson @@ -1 +1 @@ -{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.10.4","id": "8.10.4","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} +{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.14.1","id": "8.14.1","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} From 7716f4aff8a4498191c5174c3f13602532e525ad Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 27 Jun 2024 10:49:52 -0400 Subject: [PATCH 06/38] Elastic 8.14.1 --- salt/kibana/tools/sbin_jinja/so-kibana-config-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/tools/sbin_jinja/so-kibana-config-load b/salt/kibana/tools/sbin_jinja/so-kibana-config-load index 5b4b52560..dd36fd030 100644 --- a/salt/kibana/tools/sbin_jinja/so-kibana-config-load +++ b/salt/kibana/tools/sbin_jinja/so-kibana-config-load @@ -63,7 +63,7 @@ update() { IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' for i in "${LINES[@]}"; do - RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.10.4" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") + RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.14.1" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi done From 92a847e3bd9e62fe6f28d9869afaa7937cae5f46 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Thu, 27 Jun 2024 11:48:54 -0400 Subject: [PATCH 07/38] Fix Fleet setup --- .../tools/sbin_jinja/so-elastic-fleet-setup | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index 0748557fd..7e497f6f5 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -53,7 +53,8 @@ fi printf "\n### Create ES Token ###\n" ESTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/service_tokens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq -r .value) -### Create Outputs & Fleet URLs ### +### Create Outputs, Fleet Policy and Fleet URLs ### +# Create the Manager Elasticsearch Output first and set it as the default output printf "\nAdd Manager Elasticsearch Output...\n" ESCACRT=$(openssl x509 -in $INTCA) JSON_STRING=$( jq -n \ @@ -62,7 +63,13 @@ JSON_STRING=$( jq -n \ curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" printf "\n\n" -printf "\nCreate Logstash Output Config if node is not an Import or Eval install\n" +# Create the Manager Fleet Server Host Agent Policy +# This has to be done while the Elasticsearch Output is set to the default Output +printf "Create Manager Fleet Server Policy...\n" +elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "true" "120" + +# Now we can create the Logstash Output and set it to to be the default Output +printf "\n\nCreate Logstash Output Config if node is not an Import or Eval install\n" {% if grains.role not in ['so-import', 'so-eval'] %} LOGSTASHCRT=$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt) LOGSTASHKEY=$(openssl rsa -in /etc/pki/elasticfleet-logstash.key) @@ -101,16 +108,6 @@ printf "\n\n" # Load Elasticsearch templates /usr/sbin/so-elasticsearch-templates-load -# Manager Fleet Server Host -elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "true" "120" - -#Temp Fixup for ES Output bug -JSON_STRING=$( jq -n \ - --arg NAME "FleetServer_{{ GLOBALS.hostname }}" \ - '{"name": $NAME,"description": $NAME,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":120,"data_output_id":"so-manager_elasticsearch"}' - ) -curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/FleetServer_{{ GLOBALS.hostname }}" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" - # Initial Endpoints Policy elastic_fleet_policy_create "endpoints-initial" "Initial Endpoint Policy" "false" "1209600" From 32d7927a495e79710c23852d108c5e04e5a30dfc Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 1 Jul 2024 15:16:06 +0000 Subject: [PATCH 08/38] Template changes for Elastic 8.14.1 --- salt/elasticsearch/defaults.yaml | 2664 ++++++++++++++++++------------ 1 file changed, 1578 insertions(+), 1086 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 36d673d70..8ca79f01e 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1,6 +1,4 @@ elasticsearch: - enabled: false - index_clean: true config: action: destructive_requires_name: true @@ -56,87 +54,8 @@ elasticsearch: enabled: true key: /usr/share/elasticsearch/config/elasticsearch.key verification_mode: none - pipelines: - custom001: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom001 - - pipeline: - name: common - custom002: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom002 - - pipeline: - name: common - custom003: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom003 - - pipeline: - name: common - custom004: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom004 - - pipeline: - name: common - custom005: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom005 - - pipeline: - name: common - custom006: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom006 - - pipeline: - name: common - custom007: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom007 - - pipeline: - name: common - custom008: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom008 - - pipeline: - name: common - custom009: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom009 - - pipeline: - name: common - custom010: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom010 - - pipeline: - name: common + enabled: false + index_clean: true index_settings: global_overrides: index_template: @@ -170,84 +89,13 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-items: - index_sorting: false - index_template: - composed_of: - - so-items-mappings - index_patterns: - - .items-default-** - priority: 500 - template: - mappings: - date_detection: false - settings: - index: - lifecycle: - name: so-items-logs - rollover_alias: ".items-default" - routing: - allocation: - include: - _tier_preference: "data_content" - mapping: - total_fields: - limit: 10000 - number_of_replicas: 0 - number_of_shards: 1 - refresh_interval: 30s - sort: - field: '@timestamp' - order: desc - policy: - phases: - hot: - actions: - rollover: - max_size: 50gb - min_age: 0ms - so-lists: - index_sorting: false - index_template: - composed_of: - - so-lists-mappings - index_patterns: - - .lists-default-** - priority: 500 - template: - mappings: - date_detection: false - settings: - index: - lifecycle: - name: so-lists-logs - rollover_alias: ".lists-default" - routing: - allocation: - include: - _tier_preference: "data_content" - mapping: - total_fields: - limit: 10000 - number_of_replicas: 0 - number_of_shards: 1 - refresh_interval: 30s - sort: - field: '@timestamp' - order: desc - policy: - phases: - hot: - actions: - rollover: - max_size: 50gb - min_age: 0ms so-case: index_sorting: false index_template: composed_of: - case-mappings - case-settings + ignore_missing_component_templates: [] index_patterns: - so-case* priority: 500 @@ -271,142 +119,6 @@ elasticsearch: sort: field: '@timestamp' order: desc - so-detection: - index_sorting: false - index_template: - composed_of: - - detection-mappings - - detection-settings - index_patterns: - - so-detection* - priority: 500 - template: - mappings: - date_detection: false - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - settings: - index: - mapping: - total_fields: - limit: 1500 - number_of_replicas: 0 - number_of_shards: 1 - refresh_interval: 30s - sort: - field: '@timestamp' - order: desc - so-logs-soc: - close: 30 - delete: 365 - index_sorting: false - index_template: - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - container-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - common-settings - - common-dynamic-mappings - data_stream: {} - index_patterns: - - logs-soc-so* - priority: 500 - template: - mappings: - date_detection: false - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - settings: - index: - lifecycle: - name: so-soc-logs - mapping: - total_fields: - limit: 5000 - number_of_replicas: 0 - number_of_shards: 1 - refresh_interval: 30s - sort: - field: '@timestamp' - order: desc - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - warm: 7 so-common: close: 30 delete: 365 @@ -473,6 +185,7 @@ elasticsearch: - common-dynamic-mappings - winlog-mappings data_stream: {} + ignore_missing_component_templates: [] index_patterns: - logs-*-so* priority: 1 @@ -523,6 +236,36 @@ elasticsearch: priority: 50 min_age: 30d warm: 7 + so-detection: + index_sorting: false + index_template: + composed_of: + - detection-mappings + - detection-settings + ignore_missing_component_templates: [] + index_patterns: + - so-detection* + priority: 500 + template: + mappings: + date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string + settings: + index: + mapping: + total_fields: + limit: 1500 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc so-endgame: index_sorting: false index_template: @@ -585,6 +328,7 @@ elasticsearch: - common-settings - common-dynamic-mappings - winlog-mappings + ignore_missing_component_templates: [] index_patterns: - endgame* priority: 500 @@ -690,6 +434,7 @@ elasticsearch: - dtc-user_agent-mappings - common-settings - common-dynamic-mappings + ignore_missing_component_templates: [] index_patterns: - so-idh-* priority: 500 @@ -802,6 +547,7 @@ elasticsearch: - common-dynamic-mappings - winlog-mappings data_stream: {} + ignore_missing_component_templates: [] index_patterns: - logs-import-so* priority: 500 @@ -852,6 +598,91 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-items: + index_sorting: false + index_template: + composed_of: + - so-items-mappings + ignore_missing_component_templates: [] + index_patterns: + - .items-default-** + priority: 500 + template: + mappings: + date_detection: false + settings: + index: + lifecycle: + name: so-items-logs + rollover_alias: .items-default + mapping: + total_fields: + limit: 10000 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + routing: + allocation: + include: + _tier_preference: data_content + sort: + field: '@timestamp' + order: desc + policy: + phases: + hot: + actions: + rollover: + max_size: 50gb + min_age: 0ms + so-kismet: + index_sorting: false + index_template: + composed_of: + - kismet-mappings + - source-mappings + - client-mappings + - device-mappings + - network-mappings + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: [] + index_patterns: + - logs-kismet-so* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-kismet-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-kratos: close: 30 delete: 365 @@ -911,6 +742,7 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: [] index_patterns: - logs-kratos-so* priority: 500 @@ -961,6 +793,43 @@ elasticsearch: priority: 50 min_age: 30d warm: 7 + so-lists: + index_sorting: false + index_template: + composed_of: + - so-lists-mappings + ignore_missing_component_templates: [] + index_patterns: + - .lists-default-** + priority: 500 + template: + mappings: + date_detection: false + settings: + index: + lifecycle: + name: so-lists-logs + rollover_alias: .lists-default + mapping: + total_fields: + limit: 10000 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + routing: + allocation: + include: + _tier_preference: data_content + sort: + field: '@timestamp' + order: desc + policy: + phases: + hot: + actions: + rollover: + max_size: 50gb + min_age: 0ms so-logs: index_sorting: false index_template: @@ -973,6 +842,7 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: [] index_patterns: - logs-*-* priority: 225 @@ -1034,6 +904,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-1password.item_usages@custom index_patterns: - logs-1password.item_usages-* priority: 501 @@ -1078,6 +950,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-1password.signin_attempts@custom index_patterns: - logs-1password.signin_attempts-* priority: 501 @@ -1122,6 +996,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-apache.access@custom index_patterns: - logs-apache.access-* priority: 501 @@ -1166,6 +1042,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-apache.error@custom index_patterns: - logs-apache.error-* priority: 501 @@ -1210,6 +1088,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-auditd.log@custom index_patterns: - logs-auditd.log-* priority: 501 @@ -1254,6 +1134,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-auth0.logs@custom index_patterns: - logs-auth0.logs-* priority: 501 @@ -1288,25 +1170,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-aws_x_cloudfront_logs: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-aws.cloudfront_logs@package + - logs-aws.cloudfront_logs@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-aws.cloudfront_logs@custom index_patterns: - - "logs-aws.cloudfront_logs-*" + - logs-aws.cloudfront_logs-* + priority: 501 template: settings: index: lifecycle: name: so-logs-aws.cloudfront_logs-logs number_of_replicas: 0 - composed_of: - - "logs-aws.cloudfront_logs@package" - - "logs-aws.cloudfront_logs@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -1342,6 +1226,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.cloudtrail@custom index_patterns: - logs-aws.cloudtrail-* priority: 501 @@ -1386,6 +1272,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.cloudwatch_logs@custom index_patterns: - logs-aws.cloudwatch_logs-* priority: 501 @@ -1430,6 +1318,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.ec2_logs@custom index_patterns: - logs-aws.ec2_logs-* priority: 501 @@ -1474,6 +1364,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.elb_logs@custom index_patterns: - logs-aws.elb_logs-* priority: 501 @@ -1518,6 +1410,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.firewall_logs@custom index_patterns: - logs-aws.firewall_logs-* priority: 501 @@ -1552,25 +1446,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-aws_x_guardduty: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-aws.guardduty@package + - logs-aws.guardduty@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-aws.guardduty@custom index_patterns: - - "logs-aws.guardduty-*" + - logs-aws.guardduty-* + priority: 501 template: settings: index: lifecycle: name: so-logs-aws.guardduty-logs number_of_replicas: 0 - composed_of: - - "logs-aws.guardduty@package" - - "logs-aws.guardduty@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -1596,25 +1492,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-aws_x_inspector: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-aws.inspector@package + - logs-aws.inspector@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-aws.inspector@custom index_patterns: - - "logs-aws.inspector-*" + - logs-aws.inspector-* + priority: 501 template: settings: index: lifecycle: name: so-logs-aws.inspector-logs number_of_replicas: 0 - composed_of: - - "logs-aws.inspector@package" - - "logs-aws.inspector@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -1650,6 +1548,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.route53_public_logs@custom index_patterns: - logs-aws.route53_public_logs-* priority: 501 @@ -1694,6 +1594,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.route53_resolver_logs@custom index_patterns: - logs-aws.route53_resolver_logs-* priority: 501 @@ -1738,6 +1640,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.s3access@custom index_patterns: - logs-aws.s3access-* priority: 501 @@ -1772,25 +1676,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-aws_x_securityhub_findings: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-aws.securityhub_findings@package + - logs-aws.securityhub_findings@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-aws.securityhub_findings@custom index_patterns: - - "logs-aws.securityhub_findings-*" + - logs-aws.securityhub_findings-* + priority: 501 template: settings: index: lifecycle: name: so-logs-aws.securityhub_findings-logs number_of_replicas: 0 - composed_of: - - "logs-aws.securityhub_findings@package" - - "logs-aws.securityhub_findings@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -1816,25 +1722,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-aws_x_securityhub_insights: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-aws.securityhub_insights@package + - logs-aws.securityhub_insights@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-aws.securityhub_insights@custom index_patterns: - - "logs-aws.securityhub_insights-*" + - logs-aws.securityhub_insights-* + priority: 501 template: settings: index: lifecycle: name: so-logs-aws.securityhub_insights-logs number_of_replicas: 0 - composed_of: - - "logs-aws.securityhub_insights@package" - - "logs-aws.securityhub_insights@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -1870,6 +1778,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.vpcflow@custom index_patterns: - logs-aws.vpcflow-* priority: 501 @@ -1914,6 +1824,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.waf@custom index_patterns: - logs-aws.waf-* priority: 501 @@ -1958,6 +1870,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.activitylogs@custom index_patterns: - logs-azure.activitylogs-* priority: 501 @@ -2002,6 +1916,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.application_gateway@custom index_patterns: - logs-azure.application_gateway-* priority: 501 @@ -2046,6 +1962,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.auditlogs@custom index_patterns: - logs-azure.auditlogs-* priority: 501 @@ -2090,6 +2008,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.eventhub@custom index_patterns: - logs-azure.eventhub-* priority: 501 @@ -2134,6 +2054,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.firewall_logs@custom index_patterns: - logs-azure.firewall_logs-* priority: 501 @@ -2178,6 +2100,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.identity_protection@custom index_patterns: - logs-azure.identity_protection-* priority: 501 @@ -2222,6 +2146,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.platformlogs@custom index_patterns: - logs-azure.platformlogs-* priority: 501 @@ -2266,6 +2192,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.provisioning@custom index_patterns: - logs-azure.provisioning-* priority: 501 @@ -2310,6 +2238,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.signinlogs@custom index_patterns: - logs-azure.signinlogs-* priority: 501 @@ -2354,6 +2284,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.springcloudlogs@custom index_patterns: - logs-azure.springcloudlogs-* priority: 501 @@ -2398,6 +2330,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-barracuda.waf@custom index_patterns: - logs-barracuda.waf-* priority: 501 @@ -2442,6 +2376,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-carbonblack_edr.log@custom index_patterns: - logs-carbonblack_edr.log-* priority: 501 @@ -2476,25 +2412,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-cef_x_log: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-cef.log@package + - logs-cef.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-cef.log@custom index_patterns: - - "logs-cef.log-*" + - logs-cef.log-* + priority: 501 template: settings: index: lifecycle: name: so-logs-cef.log-logs number_of_replicas: 0 - composed_of: - - "logs-cef.log@package" - - "logs-cef.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -2520,25 +2458,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-checkpoint_x_firewall: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-checkpoint.firewall@package + - logs-checkpoint.firewall@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-checkpoint.firewall@custom index_patterns: - - "logs-checkpoint.firewall-*" + - logs-checkpoint.firewall-* + priority: 501 template: settings: index: lifecycle: name: so-logs-checkpoint.firewall-logs number_of_replicas: 0 - composed_of: - - "logs-checkpoint.firewall@package" - - "logs-checkpoint.firewall@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -2574,6 +2514,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_asa.log@custom index_patterns: - logs-cisco_asa.log-* priority: 501 @@ -2618,6 +2560,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_duo.admin@custom index_patterns: - logs-cisco_duo.admin-* priority: 501 @@ -2662,6 +2606,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_duo.auth@custom index_patterns: - logs-cisco_duo.auth-* priority: 501 @@ -2706,6 +2652,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_duo.offline_enrollment@custom index_patterns: - logs-cisco_duo.offline_enrollment-* priority: 501 @@ -2750,6 +2698,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_duo.summary@custom index_patterns: - logs-cisco_duo.summary-* priority: 501 @@ -2794,6 +2744,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_duo.telephony@custom index_patterns: - logs-cisco_duo.telephony-* priority: 501 @@ -2828,25 +2780,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-cisco_ftd_x_log: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-cisco_ftd.log@package + - logs-cisco_ftd.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-cisco_ftd.log@custom index_patterns: - - "logs-cisco_ftd.log-*" + - logs-cisco_ftd.log-* + priority: 501 template: settings: index: lifecycle: name: so-logs-cisco_ftd.log-logs number_of_replicas: 0 - composed_of: - - "logs-cisco_ftd.log@package" - - "logs-cisco_ftd.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -2872,25 +2826,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-cisco_ios_x_log: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-cisco_ios.log@package + - logs-cisco_ios.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-cisco_ios.log@custom index_patterns: - - "logs-cisco_ios.log-*" + - logs-cisco_ios.log-* + priority: 501 template: settings: index: lifecycle: name: so-logs-cisco_ios.log-logs number_of_replicas: 0 - composed_of: - - "logs-cisco_ios.log@package" - - "logs-cisco_ios.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -2916,25 +2872,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-cisco_ise_x_log: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-cisco_ise.log@package + - logs-cisco_ise.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-cisco_ise.log@custom index_patterns: - - "logs-cisco_ise.log-*" + - logs-cisco_ise.log-* + priority: 501 template: settings: index: lifecycle: name: so-logs-cisco_ise.log-logs number_of_replicas: 0 - composed_of: - - "logs-cisco_ise.log@package" - - "logs-cisco_ise.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -2970,6 +2928,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_meraki.events@custom index_patterns: - logs-cisco_meraki.events-* priority: 501 @@ -3014,6 +2974,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_meraki.log@custom index_patterns: - logs-cisco_meraki.log-* priority: 501 @@ -3058,6 +3020,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_umbrella.log@custom index_patterns: - logs-cisco_umbrella.log-* priority: 501 @@ -3092,25 +3056,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_adc_x_interface: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-citrix_adc.interface@package + - logs-citrix_adc.interface@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-citrix_adc.interface@custom index_patterns: - - "logs-citrix_adc.interface-*" + - logs-citrix_adc.interface-* + priority: 501 template: settings: index: lifecycle: name: so-logs-citrix_adc.interface-logs number_of_replicas: 0 - composed_of: - - "logs-citrix_adc.interface@package" - - "logs-citrix_adc.interface@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -3136,25 +3102,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_adc_x_lbvserver: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-citrix_adc.lbvserver@package + - logs-citrix_adc.lbvserver@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-citrix_adc.lbvserver@custom index_patterns: - - "logs-citrix_adc.lbvserver-*" + - logs-citrix_adc.lbvserver-* + priority: 501 template: settings: index: lifecycle: name: so-logs-citrix_adc.lbvserver-logs number_of_replicas: 0 - composed_of: - - "logs-citrix_adc.lbvserver@package" - - "logs-citrix_adc.lbvserver@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -3180,25 +3148,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_adc_x_service: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-citrix_adc.service@package + - logs-citrix_adc.service@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-citrix_adc.service@custom index_patterns: - - "logs-citrix_adc.service-*" + - logs-citrix_adc.service-* + priority: 501 template: settings: index: lifecycle: name: so-logs-citrix_adc.service-logs number_of_replicas: 0 - composed_of: - - "logs-citrix_adc.service@package" - - "logs-citrix_adc.service@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -3224,25 +3194,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_adc_x_system: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-citrix_adc.system@package + - logs-citrix_adc.system@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-citrix_adc.system@custom index_patterns: - - "logs-citrix_adc.system-*" + - logs-citrix_adc.system-* + priority: 501 template: settings: index: lifecycle: name: so-logs-citrix_adc.system-logs number_of_replicas: 0 - composed_of: - - "logs-citrix_adc.system@package" - - "logs-citrix_adc.system@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -3268,25 +3240,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_adc_x_vpn: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-citrix_adc.vpn@package + - logs-citrix_adc.vpn@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-citrix_adc.vpn@custom index_patterns: - - "logs-citrix_adc.vpn-*" + - logs-citrix_adc.vpn-* + priority: 501 template: settings: index: lifecycle: name: so-logs-citrix_adc.vpn-logs number_of_replicas: 0 - composed_of: - - "logs-citrix_adc.vpn@package" - - "logs-citrix_adc.vpn@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -3312,25 +3286,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_waf_x_log: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-citrix_waf.log@package + - logs-citrix_waf.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-citrix_waf.log@custom index_patterns: - - "logs-citrix_waf.log-*" + - logs-citrix_waf.log-* + priority: 501 template: settings: index: lifecycle: name: so-logs-citrix_waf.log-logs number_of_replicas: 0 - composed_of: - - "logs-citrix_waf.log@package" - - "logs-citrix_waf.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -3366,6 +3342,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cloudflare.audit@custom index_patterns: - logs-cloudflare.audit-* priority: 501 @@ -3410,6 +3388,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cloudflare.logpull@custom index_patterns: - logs-cloudflare.logpull-* priority: 501 @@ -3454,6 +3434,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-crowdstrike.falcon@custom index_patterns: - logs-crowdstrike.falcon-* priority: 501 @@ -3498,6 +3480,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-crowdstrike.fdr@custom index_patterns: - logs-crowdstrike.fdr-* priority: 501 @@ -3542,6 +3526,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-darktrace.ai_analyst_alert@custom index_patterns: - logs-darktrace.ai_analyst_alert-* priority: 501 @@ -3586,6 +3572,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-darktrace.model_breach_alert@custom index_patterns: - logs-darktrace.model_breach_alert-* priority: 501 @@ -3630,6 +3618,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-darktrace.system_status_alert@custom index_patterns: - logs-darktrace.system_status_alert-* priority: 501 @@ -3675,6 +3665,7 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: [] index_patterns: - logs-detections.alerts-* priority: 501 @@ -3737,6 +3728,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent@custom index_patterns: - logs-elastic_agent-* priority: 501 @@ -3798,6 +3791,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent.apm_server@custom index_patterns: - logs-elastic_agent.apm_server-* priority: 501 @@ -3859,6 +3854,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent.auditbeat@custom index_patterns: - logs-elastic_agent.auditbeat-* priority: 501 @@ -3917,6 +3914,8 @@ elasticsearch: - logs-elastic_agent.cloudbeat@custom - so-fleet_globals-1 - so-fleet_agent_id_verification-1 + ignore_missing_component_templates: + - logs-elastic_agent.cloudbeat@custom index_patterns: - logs-elastic_agent.cloudbeat-* priority: 501 @@ -3979,6 +3978,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent.endpoint_security@custom index_patterns: - logs-elastic_agent.endpoint_security-* priority: 501 @@ -4035,6 +4036,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent.filebeat@custom index_patterns: - logs-elastic_agent.filebeat-* priority: 501 @@ -4091,6 +4094,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent.fleet_server@custom index_patterns: - logs-elastic_agent.fleet_server-* priority: 501 @@ -4140,6 +4145,8 @@ elasticsearch: - logs-elastic_agent.heartbeat@custom - so-fleet_globals-1 - so-fleet_agent_id_verification-1 + ignore_missing_component_templates: + - logs-elastic_agent.heartbeat@custom index_patterns: - logs-elastic_agent.heartbeat-* priority: 501 @@ -4202,6 +4209,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent.metricbeat@custom index_patterns: - logs-elastic_agent.metricbeat-* priority: 501 @@ -4258,6 +4267,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent.osquerybeat@custom index_patterns: - logs-elastic_agent.osquerybeat-* priority: 501 @@ -4313,6 +4324,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent.packetbeat@custom index_patterns: - logs-elastic_agent.packetbeat-* priority: 501 @@ -4375,6 +4388,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.alerts@custom index_patterns: - logs-endpoint.alerts-* priority: 501 @@ -4431,6 +4446,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.diagnostic.collection@custom index_patterns: - .logs-endpoint.diagnostic.collection-* priority: 501 @@ -4487,6 +4504,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.events.api@custom index_patterns: - logs-endpoint.events.api-* priority: 501 @@ -4543,6 +4562,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.events.file@custom index_patterns: - logs-endpoint.events.file-* priority: 501 @@ -4599,6 +4620,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.events.library@custom index_patterns: - logs-endpoint.events.library-* priority: 501 @@ -4655,6 +4678,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.events.network@custom index_patterns: - logs-endpoint.events.network-* priority: 501 @@ -4711,6 +4736,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.events.process@custom index_patterns: - logs-endpoint.events.process-* priority: 501 @@ -4767,6 +4794,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.events.registry@custom index_patterns: - logs-endpoint.events.registry-* priority: 501 @@ -4823,6 +4852,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.events.security@custom index_patterns: - logs-endpoint.events.security-* priority: 501 @@ -4878,6 +4909,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-f5_bigip.log@custom index_patterns: - logs-f5_bigip.log-* priority: 501 @@ -4922,6 +4955,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-fim.event@custom index_patterns: - logs-fim.event-* priority: 501 @@ -4966,6 +5001,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-fireeye.nx@custom index_patterns: - logs-fireeye.nx-* priority: 501 @@ -5010,6 +5047,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-fortinet_fortigate.log@custom index_patterns: - logs-fortinet_fortigate.log-* priority: 501 @@ -5054,6 +5093,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-fortinet.clientendpoint@custom index_patterns: - logs-fortinet.clientendpoint-* priority: 501 @@ -5098,6 +5139,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-fortinet.firewall@custom index_patterns: - logs-fortinet.firewall-* priority: 501 @@ -5142,6 +5185,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-fortinet.fortimail@custom index_patterns: - logs-fortinet.fortimail-* priority: 501 @@ -5186,6 +5231,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-fortinet.fortimanager@custom index_patterns: - logs-fortinet.fortimanager-* priority: 501 @@ -5230,6 +5277,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-gcp.audit@custom index_patterns: - logs-gcp.audit-* priority: 501 @@ -5274,6 +5323,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-gcp.dns@custom index_patterns: - logs-gcp.dns-* priority: 501 @@ -5318,6 +5369,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-gcp.firewall@custom index_patterns: - logs-gcp.firewall-* priority: 501 @@ -5362,6 +5415,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-gcp.loadbalancing_logs@custom index_patterns: - logs-gcp.loadbalancing_logs-* priority: 501 @@ -5406,6 +5461,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-gcp.vpcflow@custom index_patterns: - logs-gcp.vpcflow-* priority: 501 @@ -5450,6 +5507,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-github.audit@custom index_patterns: - logs-github.audit-* priority: 501 @@ -5494,6 +5553,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-github.code_scanning@custom index_patterns: - logs-github.code_scanning-* priority: 501 @@ -5538,6 +5599,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-github.dependabot@custom index_patterns: - logs-github.dependabot-* priority: 501 @@ -5582,6 +5645,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-github.issues@custom index_patterns: - logs-github.issues-* priority: 501 @@ -5626,6 +5691,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-github.secret_scanning@custom index_patterns: - logs-github.secret_scanning-* priority: 501 @@ -5670,6 +5737,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.access_transparency@custom index_patterns: - logs-google_workspace.access_transparency-* priority: 501 @@ -5714,6 +5783,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.admin@custom index_patterns: - logs-google_workspace.admin-* priority: 501 @@ -5758,6 +5829,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.alert@custom index_patterns: - logs-google_workspace.alert-* priority: 501 @@ -5802,6 +5875,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.context_aware_access@custom index_patterns: - logs-google_workspace.context_aware_access-* priority: 501 @@ -5846,6 +5921,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.device@custom index_patterns: - logs-google_workspace.device-* priority: 501 @@ -5890,6 +5967,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.drive@custom index_patterns: - logs-google_workspace.drive-* priority: 501 @@ -5934,6 +6013,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.gcp@custom index_patterns: - logs-google_workspace.gcp-* priority: 501 @@ -5978,6 +6059,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.group_enterprise@custom index_patterns: - logs-google_workspace.group_enterprise-* priority: 501 @@ -6022,6 +6105,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.groups@custom index_patterns: - logs-google_workspace.groups-* priority: 501 @@ -6066,6 +6151,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.login@custom index_patterns: - logs-google_workspace.login-* priority: 501 @@ -6110,6 +6197,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.rules@custom index_patterns: - logs-google_workspace.rules-* priority: 501 @@ -6154,6 +6243,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.saml@custom index_patterns: - logs-google_workspace.saml-* priority: 501 @@ -6198,6 +6289,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.token@custom index_patterns: - logs-google_workspace.token-* priority: 501 @@ -6242,6 +6335,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.user_accounts@custom index_patterns: - logs-google_workspace.user_accounts-* priority: 501 @@ -6286,6 +6381,9 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-http_endpoint.generic@package + - logs-http_endpoint.generic@custom index_patterns: - logs-http_endpoint.generic-* priority: 501 @@ -6330,6 +6428,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-httpjson.generic@custom index_patterns: - logs-httpjson.generic-* priority: 501 @@ -6364,25 +6464,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-iis_x_access: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-iis.access@package + - logs-iis.access@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-iis.access@custom index_patterns: - - "logs-iis.access-*" + - logs-iis.access-* + priority: 501 template: settings: index: lifecycle: name: so-logs-iis.access-logs number_of_replicas: 0 - composed_of: - - "logs-iis.access@package" - - "logs-iis.access@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -6408,25 +6510,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-iis_x_error: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-iis.error@package + - logs-iis.error@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-iis.error@custom index_patterns: - - "logs-iis.error-*" + - logs-iis.error-* + priority: 501 template: settings: index: lifecycle: name: so-logs-iis.error-logs number_of_replicas: 0 - composed_of: - - "logs-iis.error@package" - - "logs-iis.error@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -6462,6 +6566,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-juniper_srx.log@custom index_patterns: - logs-juniper_srx.log-* priority: 501 @@ -6506,6 +6612,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-juniper.junos@custom index_patterns: - logs-juniper.junos-* priority: 501 @@ -6550,6 +6658,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-juniper.netscreen@custom index_patterns: - logs-juniper.netscreen-* priority: 501 @@ -6594,6 +6704,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-juniper.srx@custom index_patterns: - logs-juniper.srx-* priority: 501 @@ -6638,6 +6750,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-kafka_log.generic@custom index_patterns: - logs-kafka_log.generic-* priority: 501 @@ -6682,6 +6796,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-lastpass.detailed_shared_folder@custom index_patterns: - logs-lastpass.detailed_shared_folder-* priority: 501 @@ -6726,6 +6842,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-lastpass.event_report@custom index_patterns: - logs-lastpass.event_report-* priority: 501 @@ -6770,6 +6888,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-lastpass.user@custom index_patterns: - logs-lastpass.user-* priority: 501 @@ -6814,6 +6934,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-m365_defender.event@custom index_patterns: - logs-m365_defender.event-* priority: 501 @@ -6858,6 +6980,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-m365_defender.incident@custom index_patterns: - logs-m365_defender.incident-* priority: 501 @@ -6902,6 +7026,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-m365_defender.log@custom index_patterns: - logs-m365_defender.log-* priority: 501 @@ -6946,6 +7072,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-microsoft_defender_endpoint.log@custom index_patterns: - logs-microsoft_defender_endpoint.log-* priority: 501 @@ -6990,6 +7118,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-microsoft_dhcp.log@custom index_patterns: - logs-microsoft_dhcp.log-* priority: 501 @@ -7024,25 +7154,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-microsoft_sqlserver_x_audit: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-microsoft_sqlserver.audit@package + - logs-microsoft_sqlserver.audit@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-microsoft_sqlserver.audit@custom index_patterns: - - "logs-microsoft_sqlserver.audit-*" + - logs-microsoft_sqlserver.audit-* + priority: 501 template: settings: index: lifecycle: name: so-logs-microsoft_sqlserver.audit-logs number_of_replicas: 0 - composed_of: - - "logs-microsoft_sqlserver.audit@package" - - "logs-microsoft_sqlserver.audit@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -7068,113 +7200,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-microsoft_sqlserver_x_log: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-microsoft_sqlserver.log@package + - logs-microsoft_sqlserver.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-microsoft_sqlserver.log@custom index_patterns: - - "logs-microsoft_sqlserver.log-*" + - logs-microsoft_sqlserver.log-* + priority: 501 template: settings: index: lifecycle: name: so-logs-microsoft_sqlserver.log-logs number_of_replicas: 0 - composed_of: - - "logs-microsoft_sqlserver.log@package" - - "logs-microsoft_sqlserver.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-logs-mysql_x_error: - index_sorting: False - index_template: - index_patterns: - - "logs-mysql.error-*" - template: - settings: - index: - lifecycle: - name: so-logs-mysql.error-logs - number_of_replicas: 0 - composed_of: - - "logs-mysql.error@package" - - "logs-mysql.error@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-logs-mysql_x_slowlog: - index_sorting: False - index_template: - index_patterns: - - "logs-mysql.slowlog-*" - template: - settings: - index: - lifecycle: - name: so-logs-mysql.slowlog-logs - number_of_replicas: 0 - composed_of: - - "logs-mysql.slowlog@package" - - "logs-mysql.slowlog@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -7210,6 +7256,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-mimecast.audit_events@custom index_patterns: - logs-mimecast.audit_events-* priority: 501 @@ -7254,6 +7302,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-mimecast.dlp_logs@custom index_patterns: - logs-mimecast.dlp_logs-* priority: 501 @@ -7298,6 +7348,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-mimecast.siem_logs@custom index_patterns: - logs-mimecast.siem_logs-* priority: 501 @@ -7342,6 +7394,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-mimecast.threat_intel_malware_customer@custom index_patterns: - logs-mimecast.threat_intel_malware_customer-* priority: 501 @@ -7386,6 +7440,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-mimecast.threat_intel_malware_grid@custom index_patterns: - logs-mimecast.threat_intel_malware_grid-* priority: 501 @@ -7430,6 +7486,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-mimecast.ttp_ap_logs@custom index_patterns: - logs-mimecast.ttp_ap_logs-* priority: 501 @@ -7474,6 +7532,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-mimecast.ttp_ip_logs@custom index_patterns: - logs-mimecast.ttp_ip_logs-* priority: 501 @@ -7518,6 +7578,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-mimecast.ttp_url_logs@custom index_patterns: - logs-mimecast.ttp_url_logs-* priority: 501 @@ -7551,6 +7613,98 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-mysql_x_error: + index_sorting: false + index_template: + composed_of: + - logs-mysql.error@package + - logs-mysql.error@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-mysql.error@custom + index_patterns: + - logs-mysql.error-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-mysql.error-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-mysql_x_slowlog: + index_sorting: false + index_template: + composed_of: + - logs-mysql.slowlog@package + - logs-mysql.slowlog@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-mysql.slowlog@custom + index_patterns: + - logs-mysql.slowlog-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-mysql.slowlog-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-netflow_x_log: index_sorting: false index_template: @@ -7562,6 +7716,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-netflow.log@custom index_patterns: - logs-netflow.log-* priority: 501 @@ -7596,25 +7752,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-nginx_x_access: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-nginx.access@package + - logs-nginx.access@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-nginx.access@custom index_patterns: - - "logs-nginx.access-*" + - logs-nginx.access-* + priority: 501 template: settings: index: lifecycle: name: so-logs-nginx.access-logs number_of_replicas: 0 - composed_of: - - "logs-nginx.access@package" - - "logs-nginx.access@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -7640,69 +7798,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-nginx_x_error: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-nginx.error@package + - logs-nginx.error@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-nginx.error@custom index_patterns: - - "logs-nginx.error-*" + - logs-nginx.error-* + priority: 501 template: settings: index: lifecycle: name: so-logs-nginx.error-logs number_of_replicas: 0 - composed_of: - - "logs-nginx.error@package" - - "logs-nginx.error@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-nginx_x_stubstatus: - index_sorting: False - index_template: - index_patterns: - - "metrics-nginx.stubstatus-*" - template: - settings: - index: - lifecycle: - name: so-metrics-nginx.stubstatus-logs - number_of_replicas: 0 - composed_of: - - "metrics-nginx.stubstatus@package" - - "metrics-nginx.stubstatus@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -7738,6 +7854,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-o365.audit@custom index_patterns: - logs-o365.audit-* priority: 501 @@ -7782,6 +7900,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-okta.system@custom index_patterns: - logs-okta.system-* priority: 501 @@ -7825,6 +7945,7 @@ elasticsearch: name: elastic_agent composed_of: - logs-osquery_manager.action.responses + ignore_missing_component_templates: [] index_patterns: - .logs-osquery_manager.action.responses* priority: 501 @@ -7842,6 +7963,7 @@ elasticsearch: name: elastic_agent composed_of: - logs-osquery_manager.actions + ignore_missing_component_templates: [] index_patterns: - .logs-osquery_manager.actions* priority: 501 @@ -7860,6 +7982,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-panw.panos@custom index_patterns: - logs-panw.panos-* priority: 501 @@ -7904,6 +8028,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-pfsense.log@custom index_patterns: - logs-pfsense.log-* priority: 501 @@ -7938,25 +8064,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-proofpoint_tap_x_clicks_blocked: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-proofpoint_tap.clicks_blocked@package + - logs-proofpoint_tap.clicks_blocked@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-proofpoint_tap.clicks_blocked@custom index_patterns: - - "logs-proofpoint_tap.clicks_blocked-*" + - logs-proofpoint_tap.clicks_blocked-* + priority: 501 template: settings: index: lifecycle: name: so-logs-proofpoint_tap.clicks_blocked-logs number_of_replicas: 0 - composed_of: - - "logs-proofpoint_tap.clicks_blocked@package" - - "logs-proofpoint_tap.clicks_blocked@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -7982,25 +8110,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-proofpoint_tap_x_clicks_permitted: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-proofpoint_tap.clicks_permitted@package + - logs-proofpoint_tap.clicks_permitted@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-proofpoint_tap.clicks_permitted@custom index_patterns: - - "logs-proofpoint_tap.clicks_permitted-*" + - logs-proofpoint_tap.clicks_permitted-* + priority: 501 template: settings: index: lifecycle: name: so-logs-proofpoint_tap.clicks_permitted-logs number_of_replicas: 0 - composed_of: - - "logs-proofpoint_tap.clicks_permitted@package" - - "logs-proofpoint_tap.clicks_permitted@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -8026,25 +8156,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-proofpoint_tap_x_message_blocked: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-proofpoint_tap.message_blocked@package + - logs-proofpoint_tap.message_blocked@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-proofpoint_tap.message_blocked@custom index_patterns: - - "logs-proofpoint_tap.message_blocked-*" + - logs-proofpoint_tap.message_blocked-* + priority: 501 template: settings: index: lifecycle: name: so-logs-proofpoint_tap.message_blocked-logs number_of_replicas: 0 - composed_of: - - "logs-proofpoint_tap.message_blocked@package" - - "logs-proofpoint_tap.message_blocked@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -8070,25 +8202,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-proofpoint_tap_x_message_delivered: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-proofpoint_tap.message_delivered@package + - logs-proofpoint_tap.message_delivered@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-proofpoint_tap.message_delivered@custom index_patterns: - - "logs-proofpoint_tap.message_delivered-*" + - logs-proofpoint_tap.message_delivered-* + priority: 501 template: settings: index: lifecycle: name: so-logs-proofpoint_tap.message_delivered-logs number_of_replicas: 0 - composed_of: - - "logs-proofpoint_tap.message_delivered@package" - - "logs-proofpoint_tap.message_delivered@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -8124,6 +8258,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-pulse_connect_secure.log@custom index_patterns: - logs-pulse_connect_secure.log-* priority: 501 @@ -8168,6 +8304,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sentinel_one.activity@custom index_patterns: - logs-sentinel_one.activity-* priority: 501 @@ -8212,6 +8350,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sentinel_one.agent@custom index_patterns: - logs-sentinel_one.agent-* priority: 501 @@ -8256,6 +8396,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sentinel_one.alert@custom index_patterns: - logs-sentinel_one.alert-* priority: 501 @@ -8300,6 +8442,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sentinel_one.group@custom index_patterns: - logs-sentinel_one.group-* priority: 501 @@ -8344,6 +8488,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sentinel_one.threat@custom index_patterns: - logs-sentinel_one.threat-* priority: 501 @@ -8378,25 +8524,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-snort_x_log: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-snort.log@package + - logs-snort.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-snort.log@custom index_patterns: - - "logs-snort.log-*" + - logs-snort.log-* + priority: 501 template: settings: index: lifecycle: name: so-logs-snort.log-logs number_of_replicas: 0 - composed_of: - - "logs-snort.log@package" - - "logs-snort.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -8432,6 +8580,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-snyk.audit@custom index_patterns: - logs-snyk.audit-* priority: 501 @@ -8476,6 +8626,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-snyk.vulnerabilities@custom index_patterns: - logs-snyk.vulnerabilities-* priority: 501 @@ -8509,6 +8661,114 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-soc: + close: 30 + delete: 365 + index_sorting: false + index_template: + composed_of: + - agent-mappings + - dtc-agent-mappings + - base-mappings + - dtc-base-mappings + - client-mappings + - dtc-client-mappings + - container-mappings + - destination-mappings + - dtc-destination-mappings + - pb-override-destination-mappings + - dll-mappings + - dns-mappings + - dtc-dns-mappings + - ecs-mappings + - dtc-ecs-mappings + - error-mappings + - event-mappings + - dtc-event-mappings + - file-mappings + - dtc-file-mappings + - group-mappings + - host-mappings + - dtc-host-mappings + - http-mappings + - dtc-http-mappings + - log-mappings + - network-mappings + - dtc-network-mappings + - observer-mappings + - dtc-observer-mappings + - organization-mappings + - package-mappings + - process-mappings + - dtc-process-mappings + - related-mappings + - rule-mappings + - dtc-rule-mappings + - server-mappings + - service-mappings + - dtc-service-mappings + - source-mappings + - dtc-source-mappings + - pb-override-source-mappings + - threat-mappings + - tls-mappings + - url-mappings + - user_agent-mappings + - dtc-user_agent-mappings + - common-settings + - common-dynamic-mappings + data_stream: {} + ignore_missing_component_templates: [] + index_patterns: + - logs-soc-so* + priority: 500 + template: + mappings: + date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string + settings: + index: + lifecycle: + name: so-soc-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + warm: 7 so-logs-sonicwall_firewall_x_log: index_sorting: false index_template: @@ -8520,6 +8780,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sonicwall_firewall.log@custom index_patterns: - logs-sonicwall_firewall.log-* priority: 501 @@ -8564,6 +8826,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sophos_central.alert@custom index_patterns: - logs-sophos_central.alert-* priority: 501 @@ -8608,6 +8872,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sophos_central.event@custom index_patterns: - logs-sophos_central.event-* priority: 501 @@ -8652,6 +8918,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sophos.utm@custom index_patterns: - logs-sophos.utm-* priority: 501 @@ -8696,6 +8964,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sophos.xg@custom index_patterns: - logs-sophos.xg-* priority: 501 @@ -8740,6 +9010,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-symantec_endpoint.log@custom index_patterns: - logs-symantec_endpoint.log-* priority: 501 @@ -8785,6 +9057,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-system.application@custom index_patterns: - logs-system.application* priority: 501 @@ -8830,6 +9104,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-system.auth@custom index_patterns: - logs-system.auth* priority: 501 @@ -8875,6 +9151,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-system.security@custom index_patterns: - logs-system.security* priority: 501 @@ -8920,6 +9198,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-system.syslog@custom index_patterns: - logs-system.syslog* priority: 501 @@ -8965,6 +9245,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-system.system@custom index_patterns: - logs-system.system* priority: 501 @@ -9009,6 +9291,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-tenable_sc.asset@custom index_patterns: - logs-tenable_sc.asset-* priority: 501 @@ -9053,6 +9337,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-tenable_sc.plugin@custom index_patterns: - logs-tenable_sc.plugin-* priority: 501 @@ -9097,6 +9383,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-tenable_sc.vulnerability@custom index_patterns: - logs-tenable_sc.vulnerability-* priority: 501 @@ -9141,6 +9429,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_abusech.malware@custom index_patterns: - logs-ti_abusech.malware-* priority: 501 @@ -9185,6 +9475,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_abusech.malwarebazaar@custom index_patterns: - logs-ti_abusech.malwarebazaar-* priority: 501 @@ -9229,6 +9521,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_abusech.threatfox@custom index_patterns: - logs-ti_abusech.threatfox-* priority: 501 @@ -9273,6 +9567,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_abusech.url@custom index_patterns: - logs-ti_abusech.url-* priority: 501 @@ -9307,25 +9603,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-ti_anomali_x_threatstream: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-ti_anomali.threatstream@package + - logs-ti_anomali.threatstream@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-ti_anomali.threatstream@custom index_patterns: - - "logs-ti_anomali.threatstream-*" + - logs-ti_anomali.threatstream-* + priority: 501 template: settings: index: lifecycle: name: so-logs-ti_anomali.threatstream-logs number_of_replicas: 0 - composed_of: - - "logs-ti_anomali.threatstream@package" - - "logs-ti_anomali.threatstream@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -9351,25 +9649,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-ti_cybersixgill_x_threat: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-ti_cybersixgill.threat@package + - logs-ti_cybersixgill.threat@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-ti_cybersixgill.threat@custom index_patterns: - - "logs-ti_cybersixgill.threat-*" + - logs-ti_cybersixgill.threat-* + priority: 501 template: settings: index: lifecycle: name: so-logs-ti_cybersixgill.threat-logs number_of_replicas: 0 - composed_of: - - "logs-ti_cybersixgill.threat@package" - - "logs-ti_cybersixgill.threat@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -9405,6 +9705,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_misp.threat@custom index_patterns: - logs-ti_misp.threat-* priority: 501 @@ -9449,6 +9751,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_misp.threat_attributes@custom index_patterns: - logs-ti_misp.threat_attributes-* priority: 501 @@ -9493,6 +9797,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_otx.pulses_subscribed@custom index_patterns: - logs-ti_otx.pulses_subscribed-* priority: 501 @@ -9537,6 +9843,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_otx.threat@custom index_patterns: - logs-ti_otx.threat-* priority: 501 @@ -9581,6 +9889,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_recordedfuture.latest_ioc-template@custom index_patterns: - logs-ti_recordedfuture.latest_ioc-template-* priority: 501 @@ -9625,6 +9935,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_recordedfuture.threat@custom index_patterns: - logs-ti_recordedfuture.threat-* priority: 501 @@ -9659,25 +9971,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-ti_threatq_x_threat: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-ti_threatq.threat@package + - logs-ti_threatq.threat@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-ti_threatq.threat@custom index_patterns: - - "logs-ti_threatq.threat-*" + - logs-ti_threatq.threat-* + priority: 501 template: settings: index: lifecycle: name: so-logs-ti_threatq.threat-logs number_of_replicas: 0 - composed_of: - - "logs-ti_threatq.threat@package" - - "logs-ti_threatq.threat@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -9703,25 +10017,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-vsphere_x_log: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-vsphere.log@package + - logs-vsphere.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-vsphere.log@custom index_patterns: - - "logs-vsphere.log-*" + - logs-vsphere.log-* + priority: 501 template: settings: index: lifecycle: name: so-logs-vsphere.log-logs number_of_replicas: 0 - composed_of: - - "logs-vsphere.log@package" - - "logs-vsphere.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -9757,6 +10073,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-windows.forwarded@custom index_patterns: - logs-windows.forwarded* priority: 501 @@ -9801,6 +10119,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-windows.powershell@custom index_patterns: - logs-windows.powershell-* priority: 501 @@ -9845,6 +10165,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-windows.powershell_operational@custom index_patterns: - logs-windows.powershell_operational-* priority: 501 @@ -9889,6 +10211,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-windows.sysmon_operational@custom index_patterns: - logs-windows.sysmon_operational-* priority: 501 @@ -9923,25 +10247,28 @@ elasticsearch: priority: 50 min_age: 30d so-logs-winlog_x_winlog: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-winlog.winlog@package + - logs-winlog.winlog@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-winlog.winlog@package + - logs-winlog.winlog@custom index_patterns: - - "logs-winlog.winlog-*" + - logs-winlog.winlog-* + priority: 501 template: settings: index: lifecycle: name: so-logs-winlog.winlog-logs number_of_replicas: 0 - composed_of: - - "logs-winlog.winlog@package" - - "logs-winlog.winlog@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -9977,6 +10304,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zia.alerts@custom index_patterns: - logs-zscaler_zia.alerts-* priority: 501 @@ -10021,6 +10350,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zia.dns@custom index_patterns: - logs-zscaler_zia.dns-* priority: 501 @@ -10065,6 +10396,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zia.firewall@custom index_patterns: - logs-zscaler_zia.firewall-* priority: 501 @@ -10109,6 +10442,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zia.tunnel@custom index_patterns: - logs-zscaler_zia.tunnel-* priority: 501 @@ -10153,6 +10488,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zia.web@custom index_patterns: - logs-zscaler_zia.web-* priority: 501 @@ -10197,6 +10534,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zpa.app_connector_status@custom index_patterns: - logs-zscaler_zpa.app_connector_status-* priority: 501 @@ -10241,6 +10580,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zpa.audit@custom index_patterns: - logs-zscaler_zpa.audit-* priority: 501 @@ -10285,6 +10626,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zpa.browser_access@custom index_patterns: - logs-zscaler_zpa.browser_access-* priority: 501 @@ -10329,6 +10672,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zpa.user_activity@custom index_patterns: - logs-zscaler_zpa.user_activity-* priority: 501 @@ -10373,6 +10718,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zpa.user_status@custom index_patterns: - logs-zscaler_zpa.user_status-* priority: 501 @@ -10406,317 +10753,6 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-metrics-endpoint_x_metadata: - index_sorting: False - index_template: - index_patterns: - - "metrics-endpoint.metadata-*" - template: - settings: - index: - lifecycle: - name: so-metrics-endpoint.metadata-logs - number_of_replicas: 0 - composed_of: - - "metrics-endpoint.metadata@package" - - "metrics-endpoint.metadata@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-endpoint_x_metrics: - index_sorting: False - index_template: - index_patterns: - - "metrics-endpoint.metrics-*" - template: - settings: - index: - lifecycle: - name: so-metrics-endpoint.metrics-logs - number_of_replicas: 0 - composed_of: - - "metrics-endpoint.metrics@package" - - "metrics-endpoint.metrics@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-endpoint_x_policy: - index_sorting: False - index_template: - index_patterns: - - "metrics-endpoint.policy-*" - template: - settings: - index: - lifecycle: - name: so-metrics-endpoint.policy-logs - number_of_replicas: 0 - composed_of: - - "metrics-endpoint.policy@package" - - "metrics-endpoint.policy@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-vsphere_x_datastore: - index_sorting: False - index_template: - index_patterns: - - "metrics-vsphere.datastore-*" - template: - settings: - index: - lifecycle: - name: so-metrics-vsphere.datastore-logs - number_of_replicas: 0 - composed_of: - - "metrics-vsphere.datastore@package" - - "metrics-vsphere.datastore@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-vsphere_x_host: - index_sorting: False - index_template: - index_patterns: - - "metrics-vsphere.host-*" - template: - settings: - index: - lifecycle: - name: so-metrics-vsphere.host-logs - number_of_replicas: 0 - composed_of: - - "metrics-vsphere.host@package" - - "metrics-vsphere.host@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-vsphere_x_virtualmachine: - index_sorting: False - index_template: - index_patterns: - - "metrics-vsphere.virtualmachine-*" - template: - settings: - index: - lifecycle: - name: so-metrics-vsphere.virtualmachine-logs - number_of_replicas: 0 - composed_of: - - "metrics-vsphere.virtualmachine@package" - - "metrics-vsphere.virtualmachine@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-kismet: - index_sorting: false - index_template: - composed_of: - - kismet-mappings - - source-mappings - - client-mappings - - device-mappings - - network-mappings - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - index_patterns: - - logs-kismet-so* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-kismet-logs - number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d so-logstash: index_sorting: false index_template: @@ -10778,6 +10814,7 @@ elasticsearch: - vulnerability-mappings - common-settings - common-dynamic-mappings + ignore_missing_component_templates: [] index_patterns: - logs-logstash-default* priority: 500 @@ -10827,6 +10864,374 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-metrics-endpoint_x_metadata: + index_sorting: false + index_template: + composed_of: + - metrics-endpoint.metadata@package + - metrics-endpoint.metadata@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics-endpoint.metadata@custom + index_patterns: + - metrics-endpoint.metadata-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-endpoint.metadata-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-endpoint_x_metrics: + index_sorting: false + index_template: + composed_of: + - metrics-endpoint.metrics@package + - metrics-endpoint.metrics@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics-endpoint.metrics@custom + index_patterns: + - metrics-endpoint.metrics-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-endpoint.metrics-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-endpoint_x_policy: + index_sorting: false + index_template: + composed_of: + - metrics-endpoint.policy@package + - metrics-endpoint.policy@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics-endpoint.policy@custom + index_patterns: + - metrics-endpoint.policy-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-endpoint.policy-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-fleet_server_x_agent_status: + index_sorting: false + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-fleet_server.agent_status@package + - metrics-fleet_server.agent_status@custom + - ecs@mappings + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics-fleet_server.agent_status@custom + index_patterns: + - metrics-fleet_server.agent_status-* + priority: 501 + template: + settings: + index: + mode: time_series + number_of_replicas: 0 + so-metrics-fleet_server_x_agent_versions: + index_sorting: false + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-fleet_server.agent_versions@package + - metrics-fleet_server.agent_versions@custom + - ecs@mappings + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics-fleet_server.agent_versions@custom + index_patterns: + - metrics-fleet_server.agent_versions-* + priority: 501 + template: + settings: + index: + mode: time_series + number_of_replicas: 0 + so-metrics-nginx_x_stubstatus: + index_sorting: false + index_template: + composed_of: + - metrics-nginx.stubstatus@package + - metrics-nginx.stubstatus@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics-nginx.stubstatus@custom + index_patterns: + - metrics-nginx.stubstatus-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-nginx.stubstatus-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-vsphere_x_datastore: + index_sorting: false + index_template: + composed_of: + - metrics-vsphere.datastore@package + - metrics-vsphere.datastore@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics-vsphere.datastore@custom + index_patterns: + - metrics-vsphere.datastore-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-vsphere.datastore-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-vsphere_x_host: + index_sorting: false + index_template: + composed_of: + - metrics-vsphere.host@package + - metrics-vsphere.host@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics-vsphere.host@custom + index_patterns: + - metrics-vsphere.host-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-vsphere.host-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-vsphere_x_virtualmachine: + index_sorting: false + index_template: + composed_of: + - metrics-vsphere.virtualmachine@package + - metrics-vsphere.virtualmachine@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics-vsphere.virtualmachine@custom + index_patterns: + - metrics-vsphere.virtualmachine-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-vsphere.virtualmachine-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-redis: index_sorting: false index_template: @@ -10888,6 +11293,7 @@ elasticsearch: - vulnerability-mappings - common-settings - common-dynamic-mappings + ignore_missing_component_templates: [] index_patterns: - logs-redis-default* priority: 500 @@ -11000,6 +11406,7 @@ elasticsearch: - common-settings - common-dynamic-mappings data_stream: {} + ignore_missing_component_templates: [] index_patterns: - logs-strelka-so* priority: 500 @@ -11111,6 +11518,7 @@ elasticsearch: - common-settings - common-dynamic-mappings data_stream: {} + ignore_missing_component_templates: [] index_patterns: - logs-suricata-so* priority: 500 @@ -11222,6 +11630,7 @@ elasticsearch: - common-settings - common-dynamic-mappings data_stream: {} + ignore_missing_component_templates: [] index_patterns: - logs-suricata.alerts-* priority: 500 @@ -11334,6 +11743,7 @@ elasticsearch: - common-settings - common-dynamic-mappings data_stream: {} + ignore_missing_component_templates: [] index_patterns: - logs-syslog-so* priority: 500 @@ -11447,6 +11857,7 @@ elasticsearch: - common-settings - common-dynamic-mappings data_stream: {} + ignore_missing_component_templates: [] index_patterns: - logs-zeek-so* priority: 500 @@ -11496,6 +11907,87 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + pipelines: + custom001: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom001 + - pipeline: + name: common + custom002: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom002 + - pipeline: + name: common + custom003: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom003 + - pipeline: + name: common + custom004: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom004 + - pipeline: + name: common + custom005: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom005 + - pipeline: + name: common + custom006: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom006 + - pipeline: + name: common + custom007: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom007 + - pipeline: + name: common + custom008: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom008 + - pipeline: + name: common + custom009: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom009 + - pipeline: + name: common + custom010: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom010 + - pipeline: + name: common retention: retention_pct: 50 so_roles: From 1b47d5c62286e2fe3276e9e4bf943254c61c403a Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 1 Jul 2024 15:16:58 +0000 Subject: [PATCH 09/38] Changes for Elastic 8.14.1 --- .../tools/sbin_jinja/so-elasticsearch-templates-load | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load index 080348522..12ef4dbf6 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -134,7 +134,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then TEMPLATE=${i::-14} COMPONENT_PATTERN=${TEMPLATE:3} MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery") - if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" ]]; then + if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ logs-http_endpoint\.generic|logs-winlog\.winlog ]]; then load_failures=$((load_failures+1)) echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures" else @@ -153,7 +153,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then cd - >/dev/null if [[ $load_failures -eq 0 ]]; then - echo "All template loaded successfully" + echo "All templates loaded successfully" touch $STATE_FILE_SUCCESS else echo "Encountered $load_failures templates that were unable to load, likely due to missing dependencies that will be available later; will retry on next highstate" From 4d499be1a83dd3695d6e7c47a19f1b2b5394d40e Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 2 Jul 2024 08:47:29 -0400 Subject: [PATCH 10/38] Change name --- .../files/integrations/endpoints-initial/windows-defender.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json b/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json index ac4394e62..ab7e0783f 100644 --- a/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json +++ b/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json @@ -11,7 +11,7 @@ "winlogs-winlog": { "enabled": true, "streams": { - "winlog.winlog": { + "winlog.winlogs": { "enabled": true, "vars": { "channel": "Microsoft-Windows-Windows Defender/Operational", From 745b6775f1ec2d5c77192dee7b0f1c72155ea4f2 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 2 Jul 2024 09:05:35 -0400 Subject: [PATCH 11/38] Change name for ILM --- salt/elasticsearch/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 8ca79f01e..98eac89f2 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -8734,7 +8734,7 @@ elasticsearch: settings: index: lifecycle: - name: so-soc-logs + name: so-logs-soc-logs mapping: total_fields: limit: 5000 From c6f6811f47d016f648c0e4e4ae1ea126c3c95eb0 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 8 Jul 2024 09:47:34 -0400 Subject: [PATCH 12/38] Elastic 8.14.2 --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 02d16ad2c..8a6effa5c 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -8,7 +8,7 @@ # Elastic agent is not managed by salt. Because of this we must store this base information in a # script that accompanies the soup system. Since so-common is one of those special soup files, # and since this same logic is required during installation, it's included in this file. -ELASTIC_AGENT_TARBALL_VERSION="8.14.1" +ELASTIC_AGENT_TARBALL_VERSION="8.14.2" ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5" ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" From ef5967844137ca24ee2d804855ad7e61fd03a322 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 8 Jul 2024 09:48:12 -0400 Subject: [PATCH 13/38] Elastic 8.14.2 --- salt/kibana/files/config_saved_objects.ndjson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/files/config_saved_objects.ndjson b/salt/kibana/files/config_saved_objects.ndjson index f3370d053..a25bee36c 100644 --- a/salt/kibana/files/config_saved_objects.ndjson +++ b/salt/kibana/files/config_saved_objects.ndjson @@ -1 +1 @@ -{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.14.1","id": "8.14.1","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} +{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.14.2","id": "8.14.2,"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} From 9504f0885ac275cd5bd324835141c7ccc56558d6 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 8 Jul 2024 09:49:07 -0400 Subject: [PATCH 14/38] Elastic 8.14.2 --- salt/kibana/tools/sbin_jinja/so-kibana-config-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/tools/sbin_jinja/so-kibana-config-load b/salt/kibana/tools/sbin_jinja/so-kibana-config-load index dd36fd030..fc0896009 100644 --- a/salt/kibana/tools/sbin_jinja/so-kibana-config-load +++ b/salt/kibana/tools/sbin_jinja/so-kibana-config-load @@ -63,7 +63,7 @@ update() { IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' for i in "${LINES[@]}"; do - RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.14.1" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") + RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.14.2" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi done From 6a396ec1aa8c49cd483ef0462b5078c8c96e56d0 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 8 Jul 2024 11:44:27 -0400 Subject: [PATCH 15/38] Fix accidental double quote removal --- salt/kibana/files/config_saved_objects.ndjson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/files/config_saved_objects.ndjson b/salt/kibana/files/config_saved_objects.ndjson index a25bee36c..50e2ba45c 100644 --- a/salt/kibana/files/config_saved_objects.ndjson +++ b/salt/kibana/files/config_saved_objects.ndjson @@ -1 +1 @@ -{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.14.2","id": "8.14.2,"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} +{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.14.2","id": "8.14.2","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} From 2dd5ff433345b253c7b03e951e35e8867950a800 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 8 Jul 2024 16:19:53 -0400 Subject: [PATCH 16/38] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7d52aac7f..38f621b25 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.0-foxtrot +2.4.90 From 8615e5d5eab8305f337be71e6ef085ace8a75bf9 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 8 Jul 2024 16:50:06 -0400 Subject: [PATCH 17/38] Move enabled and index_clean back to the top --- salt/elasticsearch/defaults.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 98eac89f2..e1a2d192f 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1,4 +1,6 @@ elasticsearch: + enabled: false + index_clean: true config: action: destructive_requires_name: true @@ -54,8 +56,6 @@ elasticsearch: enabled: true key: /usr/share/elasticsearch/config/elasticsearch.key verification_mode: none - enabled: false - index_clean: true index_settings: global_overrides: index_template: From 669f68ad88c801f64d3d8d544aed6a749f1113e6 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 9 Jul 2024 15:39:59 +0000 Subject: [PATCH 18/38] Fleet metric annotations --- salt/elasticsearch/soc_elasticsearch.yaml | 70 +++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index f56ed313e..47beb27f8 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -530,6 +530,76 @@ elasticsearch: so-strelka: *indexSettings so-syslog: *indexSettings so-zeek: *indexSettings + so-metrics-fleet_server_x_agent_status: &fleetMetricsSettings + index_sorting: + description: Sorts the index by event time, at the cost of additional processing resource consumption. + advanced: True + readonly: True + helpLink: elasticsearch.html + index_template: + ignore_missing_component_templates: + description: Ignore component templates if they aren't in Elasticsearch. + forcedType: "[]string" + multiline: True + global: True + advanced: True + readonly: True + helpLink: elasticsearch.html + index_patterns: + description: Patterns for matching multiple indices or tables. + forceType: "[]string" + multiline: True + global: True + advanced: True + readonly: True + helpLink: elasticsearch.html + template: + settings: + index: + mode: + description: Type of mode used for this index. Time series indices can be used for metrics to reduce necessary storage. + forcedType: string + global: True + advanced: True + readonly: True + helpLink: elasticsearch.html + number_of_replicas: + description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs. + forcedType: int + global: True + advanced: True + readonly: True + helpLink: elasticsearch.html + composed_of: + description: The index template is composed of these component templates. + forcedType: "[]string" + global: True + advanced: True + readonly: True + helpLink: elasticsearch.html + priority: + description: The priority of the index template. + forcedType: int + global: True + advanced: True + readonly: True + helpLink: elasticsearch.html + data_stream: + hidden: + description: Hide the data stream. + forcedType: bool + global: True + advanced: True + readonly: True + helpLink: elasticsearch.html + allow_custom_routing: + description: Allow custom routing for the data stream. + forcedType: bool + global: True + advanced: True + readonly: True + helpLink: elasticsearch.html + so-metrics-fleet_server_x_agent_versions: *fleetMetricsSettings so_roles: so-manager: &soroleSettings config: From 9716e09b83a6f9de565708da3a8ca1fb3ae7de81 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 9 Jul 2024 12:51:34 -0400 Subject: [PATCH 19/38] Temp change for testing --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 38f621b25..7d52aac7f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.90 +2.4.0-foxtrot From 7a2f01be5373f214e896cc8250ad632ee300dcd9 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 9 Jul 2024 13:58:13 -0400 Subject: [PATCH 20/38] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7d52aac7f..38f621b25 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.0-foxtrot +2.4.90 From 1f5a990b1ee9ae184fc891fea29890d834d66962 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 9 Jul 2024 18:32:06 +0000 Subject: [PATCH 21/38] Remove lines that aren't needed right now --- salt/elasticsearch/soc_elasticsearch.yaml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 47beb27f8..085aab7f0 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -539,17 +539,11 @@ elasticsearch: index_template: ignore_missing_component_templates: description: Ignore component templates if they aren't in Elasticsearch. - forcedType: "[]string" - multiline: True - global: True advanced: True readonly: True helpLink: elasticsearch.html index_patterns: description: Patterns for matching multiple indices or tables. - forceType: "[]string" - multiline: True - global: True advanced: True readonly: True helpLink: elasticsearch.html @@ -558,44 +552,32 @@ elasticsearch: index: mode: description: Type of mode used for this index. Time series indices can be used for metrics to reduce necessary storage. - forcedType: string - global: True advanced: True readonly: True helpLink: elasticsearch.html number_of_replicas: description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs. - forcedType: int - global: True advanced: True readonly: True helpLink: elasticsearch.html composed_of: description: The index template is composed of these component templates. - forcedType: "[]string" - global: True advanced: True readonly: True helpLink: elasticsearch.html priority: description: The priority of the index template. - forcedType: int - global: True advanced: True readonly: True helpLink: elasticsearch.html data_stream: hidden: description: Hide the data stream. - forcedType: bool - global: True advanced: True readonly: True helpLink: elasticsearch.html allow_custom_routing: description: Allow custom routing for the data stream. - forcedType: bool - global: True advanced: True readonly: True helpLink: elasticsearch.html From d791b23838c5edd4f407256eccd04db70dc60428 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 10 Jul 2024 11:29:09 -0400 Subject: [PATCH 22/38] Generate new Kafka truststore Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- pillar/top.sls | 2 + salt/allowed_states.map.jinja | 4 +- salt/kafka/ca.sls | 37 +++++++++++++++++++ salt/kafka/config.map.jinja | 16 +++----- salt/kafka/config.sls | 22 +++++++++-- salt/kafka/defaults.yaml | 21 ++++++----- salt/kafka/enabled.sls | 10 +++-- salt/kafka/soc_kafka.yaml | 24 +++++++++--- salt/kafka/tools/sbin_jinja/so-kafka-trust | 13 +++++++ .../config/so/0800_input_kafka.conf.jinja | 5 ++- salt/manager/tools/sbin/soup | 5 ++- setup/so-functions | 4 +- 12 files changed, 125 insertions(+), 38 deletions(-) create mode 100644 salt/kafka/ca.sls create mode 100644 salt/kafka/tools/sbin_jinja/so-kafka-trust diff --git a/pillar/top.sls b/pillar/top.sls index 14229162f..76d1a14e1 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -233,6 +233,8 @@ base: - stig.soc_stig - soc.license - kafka.nodes + - kafka.soc_kafka + - kafka.adv_kafka '*_receiver': - logstash.nodes diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index eb73e6e29..29ee968aa 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -136,7 +136,9 @@ 'firewall', 'schedule', 'docker_clean', - 'stig' + 'stig', + 'kafka.ca', + 'kafka.ssl' ], 'so-standalone': [ 'salt.master', diff --git a/salt/kafka/ca.sls b/salt/kafka/ca.sls new file mode 100644 index 000000000..f5e78ee2c --- /dev/null +++ b/salt/kafka/ca.sls @@ -0,0 +1,37 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls.split('.')[0] in allowed_states or sls in allowed_states %} +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% set KAFKATRUST = salt['pillar.get']('kafka:truststore') %} + +kafkaconfdir: + file.directory: + - name: /opt/so/conf/kafka + - user: 960 + - group: 960 + - makedirs: True + +{% if GLOBALS.is_manager %} +# Manager runs so-kafka-trust to create truststore for Kafka ssl communication +kafka_truststore: + cmd.script: + - source: salt://kafka/tools/sbin_jinja/so-kafka-trust + - template: jinja + - cwd: /opt/so + - defaults: + GLOBALS: {{ GLOBALS }} + KAFKATRUST: {{ KAFKATRUST }} +{% endif %} + +kafkacertz: + file.managed: + - name: /opt/so/conf/kafka/kafka-truststore.jks + - source: salt://kafka/files/kafka-truststore + - user: 960 + - group: 931 + +{% endif %} \ No newline at end of file diff --git a/salt/kafka/config.map.jinja b/salt/kafka/config.map.jinja index 4c408a1e7..b054e0656 100644 --- a/salt/kafka/config.map.jinja +++ b/salt/kafka/config.map.jinja @@ -7,6 +7,7 @@ {% set KAFKA_NODES_PILLAR = salt['pillar.get']('kafka:nodes') %} {% set KAFKA_PASSWORD = salt['pillar.get']('kafka:password') %} +{% set KAFKA_TRUSTPASS = salt['pillar.get']('kafka:trustpass') %} {# Create list of KRaft controllers #} {% set controllers = [] %} @@ -67,19 +68,12 @@ {% endif %} -{# If a password other than PLACEHOLDER isn't set remove it from the server.properties #} -{% if KAFKAMERGED.config.broker.ssl_x_truststore_x_password == 'PLACEHOLDER' %} -{% do KAFKAMERGED.config.broker.pop('ssl_x_truststore_x_password') %} -{% endif %} - -{% if KAFKAMERGED.config.controller.ssl_x_truststore_x_password == 'PLACEHOLDER' %} -{% do KAFKAMERGED.config.controller.pop('ssl_x_truststore_x_password') %} -{% endif %} +{# Truststore config #} +{% do KAFKAMERGED.config.broker.update({'ssl_x_truststore_x_password': KAFKA_TRUSTPASS }) %} +{% do KAFKAMERGED.config.controller.update({'ssl_x_truststore_x_password': KAFKA_TRUSTPASS }) %} +{% do KAFKAMERGED.config.client.update({'ssl_x_truststore_x_password': KAFKA_TRUSTPASS }) %} {# Client properties stuff #} -{% if KAFKAMERGED.config.client.ssl_x_truststore_x_password == 'PLACEHOLDER' %} -{% do KAFKAMERGED.config.client.pop('ssl_x_truststore_x_password') %} -{% endif %} {% do KAFKAMERGED.config.client.update({'ssl_x_keystore_x_password': KAFKA_PASSWORD }) %} {% if 'broker' in node_type %} diff --git a/salt/kafka/config.sls b/salt/kafka/config.sls index 6293ee697..1cfd1d3eb 100644 --- a/salt/kafka/config.sls +++ b/salt/kafka/config.sls @@ -7,18 +7,21 @@ {% if sls.split('.')[0] in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} -include: - - ssl - kafka_group: group.present: - name: kafka - gid: 960 -kafka: +kafka_user: user.present: - uid: 960 - gid: 960 + - home: /opt/so/conf/kafka + - createhome: False + +kafka_home_dir: + file.absent: + - name: /home/kafka kafka_sbin_tools: file.recurse: @@ -28,6 +31,17 @@ kafka_sbin_tools: - group: 960 - file_mode: 755 +kafka_sbin_jinja_tools: + file.recurse: + - name: /usr/sbin + - source: salt://kafka/tools/sbin_jinja + - user: 960 + - group: 960 + - file_mode: 755 + - template: jinja + - defaults: + GLOBALS: {{ GLOBALS }} + kafka_log_dir: file.directory: - name: /opt/so/log/kafka diff --git a/salt/kafka/defaults.yaml b/salt/kafka/defaults.yaml index ad626458f..c20d8552c 100644 --- a/salt/kafka/defaults.yaml +++ b/salt/kafka/defaults.yaml @@ -1,10 +1,11 @@ kafka: enabled: False cluster_id: - password: controllers: reset: config: + password: + trustpass: broker: advertised_x_listeners: auto_x_create_x_topics_x_enable: true @@ -30,16 +31,16 @@ kafka: ssl_x_keystore_x_location: /etc/pki/kafka.p12 ssl_x_keystore_x_type: PKCS12 ssl_x_keystore_x_password: - ssl_x_truststore_x_location: /etc/pki/java/sos/cacerts - ssl_x_truststore_x_password: PLACEHOLDER - ssl_x_truststore_x_type: PEM + ssl_x_truststore_x_location: /etc/pki/kafka-truststore.jks + ssl_x_truststore_x_type: JKS + ssl_x_truststore_x_password: transaction_x_state_x_log_x_min_x_isr: 1 transaction_x_state_x_log_x_replication_x_factor: 1 client: security_x_protocol: SSL - ssl_x_truststore_x_location: /etc/pki/java/sos/cacerts - ssl_x_truststore_x_password: PLACEHOLDER - ssl_x_truststore_x_type: PEM + ssl_x_truststore_x_location: /etc/pki/kafka-truststore.jks + ssl_x_truststore_x_type: JKS + ssl_x_truststore_x_password: ssl_x_keystore_x_location: /etc/pki/kafka.p12 ssl_x_keystore_x_type: PKCS12 ssl_x_keystore_x_password: @@ -57,6 +58,6 @@ kafka: ssl_x_keystore_x_location: /etc/pki/kafka.p12 ssl_x_keystore_x_type: PKCS12 ssl_x_keystore_x_password: - ssl_x_truststore_x_location: /etc/pki/java/sos/cacerts - ssl_x_truststore_x_password: PLACEHOLDER - ssl_x_truststore_x_type: PEM \ No newline at end of file + ssl_x_truststore_x_location: /etc/pki/kafka-truststore.jks + ssl_x_truststore_x_type: JKS + ssl_x_truststore_x_password: \ No newline at end of file diff --git a/salt/kafka/enabled.sls b/salt/kafka/enabled.sls index 0837b5af6..362f7fde3 100644 --- a/salt/kafka/enabled.sls +++ b/salt/kafka/enabled.sls @@ -17,10 +17,11 @@ {% if 'gmd' in salt['pillar.get']('features', []) %} include: - - elasticsearch.ca - - kafka.sostatus + - kafka.ca - kafka.config + - kafka.ssl - kafka.storage + - kafka.sostatus so-kafka: docker_container.running: @@ -49,7 +50,7 @@ so-kafka: {% endfor %} - binds: - /etc/pki/kafka.p12:/etc/pki/kafka.p12:ro - - /etc/pki/tls/certs/intca.crt:/etc/pki/java/sos/cacerts:ro + - /opt/so/conf/kafka/kafka-truststore.jks:/etc/pki/kafka-truststore.jks:ro - /nsm/kafka/data/:/nsm/kafka/data/:rw - /opt/so/log/kafka:/opt/kafka/logs/:rw - /opt/so/conf/kafka/server.properties:/opt/kafka/config/kraft/server.properties:ro @@ -58,6 +59,9 @@ so-kafka: {% for sc in ['server', 'client'] %} - file: kafka_kraft_{{sc}}_properties {% endfor %} + - file: kafkacertz + - require: + - file: kafkacertz delete_so-kafka_so-status.disabled: file.uncomment: diff --git a/salt/kafka/soc_kafka.yaml b/salt/kafka/soc_kafka.yaml index 0c9c8a57e..872bf51f2 100644 --- a/salt/kafka/soc_kafka.yaml +++ b/salt/kafka/soc_kafka.yaml @@ -8,19 +8,25 @@ kafka: advanced: True sensitive: True helpLink: kafka.html - password: - description: The password to use for the Kafka certificates. - sensitive: True - helpLink: kafka.html controllers: description: A comma-separated list of hostnames that will act as Kafka controllers. These hosts will be responsible for managing the Kafka cluster. Note that only manager and receiver nodes are eligible to run Kafka. This configuration needs to be set before enabling Kafka. Failure to do so may result in Kafka topics becoming unavailable requiring manual intervention to restore functionality or reset Kafka, either of which can result in data loss. - forcedType: "string" + forcedType: string helpLink: kafka.html reset: description: Disable and reset the Kafka cluster. This will remove all Kafka data including logs that may have not yet been ingested into Elasticsearch and reverts the grid to using REDIS as the global pipeline. This is useful when testing different Kafka configurations such as rearranging Kafka brokers / controllers allowing you to reset the cluster rather than manually fixing any issues arising from attempting to reassign a Kafka broker into a controller. Enter 'YES_RESET_KAFKA' and submit to disable and reset Kafka. Make any configuration changes required and re-enable Kafka when ready. This action CANNOT be reversed. advanced: True helpLink: kafka.html config: + password: + description: The password used for the Kafka certificates. + readonly: True + sensitive: True + helpLink: kafka.html + trustpass: + description: The password used for the Kafka truststore. + readonly: True + sensitive: True + helpLink: kafka.html broker: advertised_x_listeners: description: Specify the list of listeners (hostname and port) that Kafka brokers provide to clients for communication. @@ -128,6 +134,10 @@ kafka: description: The trust store file location within the Docker container. title: ssl.truststore.location helpLink: kafka.html + ssl_x_truststore_x_type: + description: The trust store file format. + title: ssl.truststore.type + helpLink: kafka.html ssl_x_truststore_x_password: description: The trust store file password. If null, the trust store file is still use, but integrity checking is disabled. Invalid for PEM format. title: ssl.truststore.password @@ -167,6 +177,10 @@ kafka: description: The trust store file location within the Docker container. title: ssl.truststore.location helpLink: kafka.html + ssl_x_truststore_x_type: + description: The trust store file format. + title: ssl.truststore.type + helpLink: kafka.html ssl_x_truststore_x_password: description: The trust store file password. If null, the trust store file is still use, but integrity checking is disabled. Invalid for PEM format. title: ssl.truststore.password diff --git a/salt/kafka/tools/sbin_jinja/so-kafka-trust b/salt/kafka/tools/sbin_jinja/so-kafka-trust new file mode 100644 index 000000000..8d404cb9a --- /dev/null +++ b/salt/kafka/tools/sbin_jinja/so-kafka-trust @@ -0,0 +1,13 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. +{% set TRUSTPASS = salt['pillar.get']('kafka:trustpass') %} + +if [ ! -f /opt/so/saltstack/local/salt/kafka/files/kafka-truststore ]; then + docker run -v /etc/pki/ca.crt:/etc/pki/ca.crt --name so-kafkatrust --user root --entrypoint /opt/java/openjdk/bin/keytool {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kafka:{{ GLOBALS.so_version }} -import -file /etc/pki/ca.crt -alias SOS -keystore /etc/pki/kafka-truststore -storepass {{ TRUSTPASS }} -storetype jks -noprompt + docker cp so-kafkatrust:/etc/pki/kafka-truststore /opt/so/saltstack/local/salt/kafka/files/kafka-truststore + docker rm so-kafkatrust +fi \ No newline at end of file diff --git a/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja b/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja index dfb246210..00dd6d530 100644 --- a/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja +++ b/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja @@ -1,4 +1,5 @@ {%- set kafka_password = salt['pillar.get']('kafka:password') %} +{%- set kafka_trustpass = salt['pillar.get']('kafka:trustpass') %} {%- set kafka_brokers = salt['pillar.get']('kafka:nodes', {}) %} {%- set brokers = [] %} @@ -22,8 +23,8 @@ input { ssl_keystore_location => '/usr/share/logstash/kafka-logstash.p12' ssl_keystore_password => '{{ kafka_password }}' ssl_keystore_type => 'PKCS12' - ssl_truststore_location => '/etc/pki/ca-trust/extracted/java/cacerts' - ssl_truststore_password => 'changeit' + ssl_truststore_location => '/etc/pki/kafka-truststore.jks' + ssl_truststore_password => '{{ kafka_trustpass }}' decorate_events => true tags => [ "elastic-agent", "input-{{ GLOBALS.hostname}}", "kafka" ] } diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index c76fe311e..019f29ebb 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -673,7 +673,10 @@ up_to_2.4.80() { } up_to_2.4.90() { - echo "Nothing to apply" + kafkatrust=$(get_random_value) + echo ' trustpass: '$kafkatrust >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls + + INSTALLEDVERSION=2.4.90 } diff --git a/setup/so-functions b/setup/so-functions index b1469b7eb..aa9eb1909 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1180,13 +1180,15 @@ kibana_pillar() { kafka_pillar() { KAFKACLUSTERID=$(get_random_value 22) KAFKAPASS=$(get_random_value) + KAFKATRUST=$(get_random_value) logCmd "mkdir -p $local_salt_dir/pillar/kafka" logCmd "touch $adv_kafka_pillar_file" logCmd "touch $kafka_pillar_file" printf '%s\n'\ "kafka:"\ " cluster_id: $KAFKACLUSTERID"\ - " password: $KAFKAPASS" > $kafka_pillar_file + " password: $KAFKAPASS"\ + " trustpass: $KAFKATRUST" > $kafka_pillar_file } logrotate_pillar() { From 8e1edd1d91e21aea68e6092f743917295d228786 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 10 Jul 2024 11:32:43 -0400 Subject: [PATCH 23/38] split Kafka ssl from ssl/init. Certs won't be generated until Kafka is enabled. Also runs some clean up for old Kafka certs Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/kafka/disabled.sls | 11 ++- salt/kafka/ssl.sls | 196 ++++++++++++++++++++++++++++++++++++++ salt/logstash/enabled.sls | 14 ++- salt/ssl/init.sls | 190 +----------------------------------- 4 files changed, 221 insertions(+), 190 deletions(-) create mode 100644 salt/kafka/ssl.sls diff --git a/salt/kafka/disabled.sls b/salt/kafka/disabled.sls index 707e953a4..79fd0c261 100644 --- a/salt/kafka/disabled.sls +++ b/salt/kafka/disabled.sls @@ -22,4 +22,13 @@ ensure_default_pipeline: - name: | /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.enabled False; /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/global/soc_global.sls global.pipeline REDIS -{% endif %} \ No newline at end of file +{% endif %} + +{# If Kafka has never been manually enabled, the 'Kafka' user does not exist. In this case certs for Kafka should not exist since they'll be owned by uid 960 #} +{% for cert in ['kafka-client.crt','kafka-client.key','kafka.crt','kafka.key','kafka-logstash.crt','kafka-logstash.key','kafka-logstash.p12','kafka.p12','elasticfleet-kafka.p8'] %} +check_kafka_cert_{{cert}}: + file.absent: + - name: /etc/pki/{{cert}} + - onlyif: stat -c %U /etc/pki/{{cert}} | grep -q UNKNOWN + - show_changes: False +{% endfor %} \ No newline at end of file diff --git a/salt/kafka/ssl.sls b/salt/kafka/ssl.sls new file mode 100644 index 000000000..c4e46ac8a --- /dev/null +++ b/salt/kafka/ssl.sls @@ -0,0 +1,196 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls in allowed_states %} +{% from 'vars/globals.map.jinja' import GLOBALS %} + +{% set kafka_password = salt['pillar.get']('kafka:password') %} + +include: + - ca.dirs + {% set global_ca_server = [] %} + {% set x509dict = salt['mine.get'](GLOBALS.manager | lower~'*', 'x509.get_pem_entries') %} + {% for host in x509dict %} + {% if 'manager' in host.split('_')|last or host.split('_')|last == 'standalone' %} + {% do global_ca_server.append(host) %} + {% endif %} + {% endfor %} + {% set ca_server = global_ca_server[0] %} + + +{% if GLOBALS.pipeline == "KAFKA" %} + +{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] %} +kafka_client_key: + x509.private_key_managed: + - name: /etc/pki/kafka-client.key + - keysize: 4096 + - backup: True + - new: True + {% if salt['file.file_exists']('/etc/pki/kafka-client.key') -%} + - prereq: + - x509: /etc/pki/kafka-client.crt + {%- endif %} + - retry: + attempts: 5 + interval: 30 + +kafka_client_crt: + x509.certificate_managed: + - name: /etc/pki/kafka-client.crt + - ca_server: {{ ca_server }} + - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} + - signing_policy: kafka + - private_key: /etc/pki/kafka-client.key + - CN: {{ GLOBALS.hostname }} + - days_remaining: 0 + - days_valid: 820 + - backup: True + - timeout: 30 + - retry: + attempts: 5 + interval: 30 + +kafka_client_key_perms: + file.managed: + - replace: False + - name: /etc/pki/kafka-client.key + - mode: 640 + - user: 960 + - group: 939 + +kafka_client_crt_perms: + file.managed: + - replace: False + - name: /etc/pki/kafka-client.crt + - mode: 640 + - user: 960 + - group: 939 +{% endif %} + +{% if grains['role'] in ['so-manager', 'so-managersearch','so-receiver', 'so-standalone'] %} +kafka_key: + x509.private_key_managed: + - name: /etc/pki/kafka.key + - keysize: 4096 + - backup: True + - new: True + {% if salt['file.file_exists']('/etc/pki/kafka.key') -%} + - prereq: + - x509: /etc/pki/kafka.crt + {%- endif %} + - retry: + attempts: 5 + interval: 30 + +kafka_crt: + x509.certificate_managed: + - name: /etc/pki/kafka.crt + - ca_server: {{ ca_server }} + - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} + - signing_policy: kafka + - private_key: /etc/pki/kafka.key + - CN: {{ GLOBALS.hostname }} + - days_remaining: 0 + - days_valid: 820 + - backup: True + - timeout: 30 + - retry: + attempts: 5 + interval: 30 + cmd.run: + - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/kafka.key -in /etc/pki/kafka.crt -export -out /etc/pki/kafka.p12 -nodes -passout pass:{{ kafka_password }}" + - onchanges: + - x509: /etc/pki/kafka.key +kafka_key_perms: + file.managed: + - replace: False + - name: /etc/pki/kafka.key + - mode: 640 + - user: 960 + - group: 939 + +kafka_crt_perms: + file.managed: + - replace: False + - name: /etc/pki/kafka.crt + - mode: 640 + - user: 960 + - group: 939 + +kafka_pkcs12_perms: + file.managed: + - replace: False + - name: /etc/pki/kafka.p12 + - mode: 640 + - user: 960 + - group: 939 +{% endif %} + +# Standalone needs kafka-logstash for automated testing. Searchnode/manager search need it for logstash to consume from Kafka. +# Manager will have cert, but be unused until a pipeline is created and logstash enabled. +{% if grains['role'] in ['so-standalone', 'so-managersearch', 'so-searchnode', 'so-manager'] %} +kafka_logstash_key: + x509.private_key_managed: + - name: /etc/pki/kafka-logstash.key + - keysize: 4096 + - backup: True + - new: True + {% if salt['file.file_exists']('/etc/pki/kafka-logstash.key') -%} + - prereq: + - x509: /etc/pki/kafka-logstash.crt + {%- endif %} + - retry: + attempts: 5 + interval: 30 + +kafka_logstash_crt: + x509.certificate_managed: + - name: /etc/pki/kafka-logstash.crt + - ca_server: {{ ca_server }} + - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} + - signing_policy: kafka + - private_key: /etc/pki/kafka-logstash.key + - CN: {{ GLOBALS.hostname }} + - days_remaining: 0 + - days_valid: 820 + - backup: True + - timeout: 30 + - retry: + attempts: 5 + interval: 30 + cmd.run: + - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/kafka-logstash.key -in /etc/pki/kafka-logstash.crt -export -out /etc/pki/kafka-logstash.p12 -nodes -passout pass:{{ kafka_password }}" + - onchanges: + - x509: /etc/pki/kafka-logstash.key + +kafka_logstash_key_perms: + file.managed: + - replace: False + - name: /etc/pki/kafka-logstash.key + - mode: 640 + - user: 931 + - group: 939 + +kafka_logstash_crt_perms: + file.managed: + - replace: False + - name: /etc/pki/kafka-logstash.crt + - mode: 640 + - user: 931 + - group: 939 + +kafka_logstash_pkcs12_perms: + file.managed: + - replace: False + - name: /etc/pki/kafka-logstash.p12 + - mode: 640 + - user: 931 + - group: 939 + +{% endif %} + +{% endif %} \ No newline at end of file diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index f95a76f13..0f44a3767 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -14,6 +14,11 @@ include: {% if GLOBALS.role not in ['so-receiver','so-fleet'] %} - elasticsearch.ca +{% endif %} +{# Kafka ca runs on nodes that can run logstash for Kafka input / output. Only when Kafka is global pipeline #} +{% if GLOBALS.role in ['so-searchnode', 'so-manager', 'so-managersearch', 'so-receiver', 'so-standalone'] and GLOBALS.pipeline == 'KAFKA' %} + - kafka.ca + - kafka.ssl {% endif %} - logstash.config - logstash.sostatus @@ -79,8 +84,9 @@ so-logstash: - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro - /opt/so/conf/ca/tls-ca-bundle.pem:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:ro {% endif %} - {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-searchnode'] %} + {% if GLOBALS.pipeline == "KAFKA" and GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-searchnode'] %} - /etc/pki/kafka-logstash.p12:/usr/share/logstash/kafka-logstash.p12:ro + - /opt/so/conf/kafka/kafka-truststore.jks:/etc/pki/kafka-truststore.jks:ro {% endif %} {% if GLOBALS.role == 'so-eval' %} - /nsm/zeek:/nsm/zeek:ro @@ -105,6 +111,9 @@ so-logstash: - file: ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }} {% endfor %} {% endfor %} + {% if GLOBALS.pipeline == 'KAFKA' and GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-searchnode'] %} + - file: kafkacertz + {% endif %} - require: {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %} - x509: etc_filebeat_crt @@ -118,6 +127,9 @@ so-logstash: - file: cacertz - file: capemz {% endif %} + {% if GLOBALS.pipeline == 'KAFKA' and GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-searchnode'] %} + - file: kafkacertz + {% endif %} delete_so-logstash_so-status.disabled: file.uncomment: diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index abcb1a559..f5be34c40 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -17,8 +17,6 @@ {% set COMMONNAME = GLOBALS.manager %} {% endif %} -{% set kafka_password = salt['pillar.get']('kafka:password') %} - {% if grains.id.split('_')|last in ['manager', 'managersearch', 'eval', 'standalone', 'import'] %} include: - ca @@ -666,7 +664,6 @@ elastickeyperms: {%- endif %} {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] %} - elasticfleet_kafka_key: x509.private_key_managed: - name: /etc/pki/elasticfleet-kafka.key @@ -696,17 +693,13 @@ elasticfleet_kafka_crt: - retry: attempts: 5 interval: 30 - cmd.run: - - name: "/usr/bin/openssl pkcs8 -in /etc/pki/elasticfleet-kafka.key -topk8 -out /etc/pki/elasticfleet-kafka.p8 -nocrypt" - - onchanges: - - x509: elasticfleet_kafka_key elasticfleet_kafka_cert_perms: file.managed: - replace: False - name: /etc/pki/elasticfleet-kafka.crt - mode: 640 - - user: 960 + - user: 947 - group: 939 elasticfleet_kafka_key_perms: @@ -714,187 +707,8 @@ elasticfleet_kafka_key_perms: - replace: False - name: /etc/pki/elasticfleet-kafka.key - mode: 640 - - user: 960 + - user: 947 - group: 939 - -elasticfleet_kafka_pkcs8_perms: - file.managed: - - replace: False - - name: /etc/pki/elasticfleet-kafka.p8 - - mode: 640 - - user: 960 - - group: 939 - -kafka_client_key: - x509.private_key_managed: - - name: /etc/pki/kafka-client.key - - keysize: 4096 - - backup: True - - new: True - {% if salt['file.file_exists']('/etc/pki/kafka-client.key') -%} - - prereq: - - x509: /etc/pki/kafka-client.crt - {%- endif %} - - retry: - attempts: 5 - interval: 30 - -kafka_client_crt: - x509.certificate_managed: - - name: /etc/pki/kafka-client.crt - - ca_server: {{ ca_server }} - - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} - - signing_policy: kafka - - private_key: /etc/pki/kafka-client.key - - CN: {{ GLOBALS.hostname }} - - days_remaining: 0 - - days_valid: 820 - - backup: True - - timeout: 30 - - retry: - attempts: 5 - interval: 30 - -kafka_client_key_perms: - file.managed: - - replace: False - - name: /etc/pki/kafka-client.key - - mode: 640 - - user: 960 - - group: 939 - -kafka_client_crt_perms: - file.managed: - - replace: False - - name: /etc/pki/kafka-client.crt - - mode: 640 - - user: 960 - - group: 939 - -{% endif %} - -{% if grains['role'] in ['so-manager', 'so-managersearch','so-receiver', 'so-standalone'] %} - -kafka_key: - x509.private_key_managed: - - name: /etc/pki/kafka.key - - keysize: 4096 - - backup: True - - new: True - {% if salt['file.file_exists']('/etc/pki/kafka.key') -%} - - prereq: - - x509: /etc/pki/kafka.crt - {%- endif %} - - retry: - attempts: 5 - interval: 30 - -kafka_crt: - x509.certificate_managed: - - name: /etc/pki/kafka.crt - - ca_server: {{ ca_server }} - - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} - - signing_policy: kafka - - private_key: /etc/pki/kafka.key - - CN: {{ GLOBALS.hostname }} - - days_remaining: 0 - - days_valid: 820 - - backup: True - - timeout: 30 - - retry: - attempts: 5 - interval: 30 - cmd.run: - - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/kafka.key -in /etc/pki/kafka.crt -export -out /etc/pki/kafka.p12 -nodes -passout pass:{{ kafka_password }}" - - onchanges: - - x509: /etc/pki/kafka.key -kafka_key_perms: - file.managed: - - replace: False - - name: /etc/pki/kafka.key - - mode: 640 - - user: 960 - - group: 939 - -kafka_crt_perms: - file.managed: - - replace: False - - name: /etc/pki/kafka.crt - - mode: 640 - - user: 960 - - group: 939 - -kafka_pkcs12_perms: - file.managed: - - replace: False - - name: /etc/pki/kafka.p12 - - mode: 640 - - user: 960 - - group: 939 - -{% endif %} - -# Standalone needs kafka-logstash for automated testing. Searchnode/manager search need it for logstash to consume from Kafka. -# Manager will have cert, but be unused until a pipeline is created and logstash enabled. -{% if grains['role'] in ['so-standalone', 'so-managersearch', 'so-searchnode', 'so-manager'] %} -kafka_logstash_key: - x509.private_key_managed: - - name: /etc/pki/kafka-logstash.key - - keysize: 4096 - - backup: True - - new: True - {% if salt['file.file_exists']('/etc/pki/kafka-logstash.key') -%} - - prereq: - - x509: /etc/pki/kafka-logstash.crt - {%- endif %} - - retry: - attempts: 5 - interval: 30 - -kafka_logstash_crt: - x509.certificate_managed: - - name: /etc/pki/kafka-logstash.crt - - ca_server: {{ ca_server }} - - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} - - signing_policy: kafka - - private_key: /etc/pki/kafka-logstash.key - - CN: {{ GLOBALS.hostname }} - - days_remaining: 0 - - days_valid: 820 - - backup: True - - timeout: 30 - - retry: - attempts: 5 - interval: 30 - cmd.run: - - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/kafka-logstash.key -in /etc/pki/kafka-logstash.crt -export -out /etc/pki/kafka-logstash.p12 -nodes -passout pass:{{ kafka_password }}" - - onchanges: - - x509: /etc/pki/kafka-logstash.key - -kafka_logstash_key_perms: - file.managed: - - replace: False - - name: /etc/pki/kafka-logstash.key - - mode: 640 - - user: 960 - - group: 939 - -kafka_logstash_crt_perms: - file.managed: - - replace: False - - name: /etc/pki/kafka-logstash.crt - - mode: 640 - - user: 960 - - group: 939 - -kafka_logstash_pkcs12_perms: - file.managed: - - replace: False - - name: /etc/pki/kafka-logstash.p12 - - mode: 640 - - user: 960 - - group: 931 - {% endif %} {% else %} From d5faf535c3adf6101d2f7ad6c1498956aeba0a10 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 10 Jul 2024 11:36:44 -0400 Subject: [PATCH 24/38] Only interact with logstash configuration when Kafka pipeline is enabled otherwise leave it default Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/kafka/defaults.yaml | 1 + salt/kafka/soc_kafka.yaml | 6 ++++++ salt/logstash/defaults.yaml | 3 +-- salt/logstash/init.sls | 6 +----- salt/logstash/map.jinja | 12 ++++++++++++ 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/salt/kafka/defaults.yaml b/salt/kafka/defaults.yaml index c20d8552c..21d6956ba 100644 --- a/salt/kafka/defaults.yaml +++ b/salt/kafka/defaults.yaml @@ -3,6 +3,7 @@ kafka: cluster_id: controllers: reset: + logstash: [] config: password: trustpass: diff --git a/salt/kafka/soc_kafka.yaml b/salt/kafka/soc_kafka.yaml index 872bf51f2..8087f9bdf 100644 --- a/salt/kafka/soc_kafka.yaml +++ b/salt/kafka/soc_kafka.yaml @@ -16,6 +16,12 @@ kafka: description: Disable and reset the Kafka cluster. This will remove all Kafka data including logs that may have not yet been ingested into Elasticsearch and reverts the grid to using REDIS as the global pipeline. This is useful when testing different Kafka configurations such as rearranging Kafka brokers / controllers allowing you to reset the cluster rather than manually fixing any issues arising from attempting to reassign a Kafka broker into a controller. Enter 'YES_RESET_KAFKA' and submit to disable and reset Kafka. Make any configuration changes required and re-enable Kafka when ready. This action CANNOT be reversed. advanced: True helpLink: kafka.html + logstash: + description: By default logstash is disabled when Kafka is enabled. This option allows you to specify any hosts you would like to re-enable logstash on alongside Kafka. + forcedType: "[]string" + multiline: True + advanced: True + helpLink: kafka.html config: password: description: The password used for the Kafka certificates. diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index b4bc885f6..9930b7bcf 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -25,7 +25,7 @@ logstash: - so/0011_input_endgame.conf - so/0012_input_elastic_agent.conf.jinja - so/0013_input_lumberjack_fleet.conf - - so/9999_output_redis.conf.jinja + - so/9999_output_redis.conf.jinja receiver: - so/0011_input_endgame.conf - so/0012_input_elastic_agent.conf.jinja @@ -35,7 +35,6 @@ logstash: - so/0900_input_redis.conf.jinja - so/9805_output_elastic_agent.conf.jinja - so/9900_output_endgame.conf.jinja - - so/0800_input_kafka.conf.jinja custom0: [] custom1: [] custom2: [] diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 3bc539b35..62b2a2ebb 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -4,13 +4,9 @@ # Elastic License 2.0. {% from 'logstash/map.jinja' import LOGSTASH_MERGED %} -{% from 'kafka/map.jinja' import KAFKAMERGED %} include: -{# Disable logstash when Kafka is enabled except when the role is standalone #} -{% if LOGSTASH_MERGED.enabled and grains.role == 'so-standalone' %} - - logstash.enabled -{% elif LOGSTASH_MERGED.enabled and not KAFKAMERGED.enabled %} +{% if LOGSTASH_MERGED.enabled %} - logstash.enabled {% else %} - logstash.disabled diff --git a/salt/logstash/map.jinja b/salt/logstash/map.jinja index 0bb6de2b7..2c3e02864 100644 --- a/salt/logstash/map.jinja +++ b/salt/logstash/map.jinja @@ -6,6 +6,7 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% import_yaml 'logstash/defaults.yaml' as LOGSTASH_DEFAULTS %} {% set LOGSTASH_MERGED = salt['pillar.get']('logstash', LOGSTASH_DEFAULTS.logstash, merge=True) %} +{% set KAFKA_LOGSTASH = salt['pillar.get']('kafka:logstash', []) %} {# used to store the redis nodes that logstash needs to know about to pull from the queue #} {% set LOGSTASH_REDIS_NODES = [] %} @@ -30,3 +31,14 @@ {% endfor %} {% endfor %} +{# Append Kafka input pipeline when Kafka is enabled #} +{% if GLOBALS.pipeline == 'KAFKA' %} +{% do LOGSTASH_MERGED.defined_pipelines.search.append('so/0800_input_kafka.conf.jinja') %} +{% do LOGSTASH_MERGED.defined_pipelines.manager.append('so/0800_input_kafka.conf.jinja') %} +{# Disable logstash on manager & receiver nodes unless it has an override configured #} +{% if not KAFKA_LOGSTASH %} +{% if GLOBALS.role in ['so-manager', 'so-receiver'] and GLOBALS.hostname not in KAFKA_LOGSTASH %} +{% do LOGSTASH_MERGED.update({'enabled': False}) %} +{% endif %} +{% endif %} +{% endif %} \ No newline at end of file From 0d8fd42be3e6c79f2a7f3e32fec01bc8291e81f2 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 10 Jul 2024 11:37:07 -0400 Subject: [PATCH 25/38] update pillarwatch engine Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/salt/files/engines.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/salt/files/engines.conf b/salt/salt/files/engines.conf index 3a9b51207..15d55e18f 100644 --- a/salt/salt/files/engines.conf +++ b/salt/salt/files/engines.conf @@ -43,20 +43,20 @@ engines: - cmd.run: cmd: /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.enabled True - cmd.run: - cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' saltutil.kill_all_jobs + cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver or G@role:so-searchnode' saltutil.kill_all_jobs - cmd.run: cmd: salt-call state.apply kafka.nodes - cmd.run: - cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' state.highstate + cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver or G@role:so-searchnode' state.highstate 'KAFKA': to: 'REDIS': - cmd.run: cmd: /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.enabled False - cmd.run: - cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' saltutil.kill_all_jobs + cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver or G@role:so-searchnode' saltutil.kill_all_jobs - cmd.run: - cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' state.highstate + cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver or G@role:so-searchnode' state.highstate - files: - /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls - /opt/so/saltstack/local/pillar/kafka/adv_kafka.sls From d432019ad91bd86f65c247ed1ee43ba89808166a Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 10 Jul 2024 12:48:08 -0400 Subject: [PATCH 26/38] Change version from 1.13.1 to 1.20.0 --- .../component/elastic-agent/logs-elastic_agent@package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json index 7bcc34de9..efd85bb4b 100644 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json +++ b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json @@ -6,7 +6,7 @@ "name": "logs" }, "codec": "best_compression", - "default_pipeline": "logs-elastic_agent-1.13.1", + "default_pipeline": "logs-elastic_agent-1.20.0", "mapping": { "total_fields": { "limit": "10000" From cfe5c1d76a160dec4e7b1888b04bd26bd06eb977 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 10 Jul 2024 13:24:02 -0400 Subject: [PATCH 27/38] remove elasticsearch.ca from receiver allowed_states. Replaced by generated kafka trust Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/allowed_states.map.jinja | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 29ee968aa..37795e9d7 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -197,7 +197,6 @@ 'schedule', 'docker_clean', 'kafka', - 'elasticsearch.ca', 'stig' ], 'so-desktop': [ From 4a88dedcb88833907aa53aec7f44cac27e66ed50 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 10 Jul 2024 16:18:46 -0400 Subject: [PATCH 28/38] Fixin kafka.ssl state and include name for kafka_user Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/kafka/config.sls | 1 + salt/kafka/ssl.sls | 21 +++++++++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/salt/kafka/config.sls b/salt/kafka/config.sls index 1cfd1d3eb..e9222388b 100644 --- a/salt/kafka/config.sls +++ b/salt/kafka/config.sls @@ -14,6 +14,7 @@ kafka_group: kafka_user: user.present: + - name: kafka - uid: 960 - gid: 960 - home: /opt/so/conf/kafka diff --git a/salt/kafka/ssl.sls b/salt/kafka/ssl.sls index c4e46ac8a..50a01b22c 100644 --- a/salt/kafka/ssl.sls +++ b/salt/kafka/ssl.sls @@ -4,10 +4,9 @@ # Elastic License 2.0. {% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls in allowed_states %} -{% from 'vars/globals.map.jinja' import GLOBALS %} - -{% set kafka_password = salt['pillar.get']('kafka:password') %} +{% if sls.split('.')[0] in allowed_states %} +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% set kafka_password = salt['pillar.get']('kafka:password') %} include: - ca.dirs @@ -20,10 +19,9 @@ include: {% endfor %} {% set ca_server = global_ca_server[0] %} - {% if GLOBALS.pipeline == "KAFKA" %} -{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] %} +{% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone'] %} kafka_client_key: x509.private_key_managed: - name: /etc/pki/kafka-client.key @@ -71,7 +69,7 @@ kafka_client_crt_perms: - group: 939 {% endif %} -{% if grains['role'] in ['so-manager', 'so-managersearch','so-receiver', 'so-standalone'] %} +{% if GLOBALS.role in ['so-manager', 'so-managersearch','so-receiver', 'so-standalone'] %} kafka_key: x509.private_key_managed: - name: /etc/pki/kafka.key @@ -132,7 +130,7 @@ kafka_pkcs12_perms: # Standalone needs kafka-logstash for automated testing. Searchnode/manager search need it for logstash to consume from Kafka. # Manager will have cert, but be unused until a pipeline is created and logstash enabled. -{% if grains['role'] in ['so-standalone', 'so-managersearch', 'so-searchnode', 'so-manager'] %} +{% if GLOBALS.role in ['so-standalone', 'so-managersearch', 'so-searchnode', 'so-manager'] %} kafka_logstash_key: x509.private_key_managed: - name: /etc/pki/kafka-logstash.key @@ -191,6 +189,13 @@ kafka_logstash_pkcs12_perms: - user: 931 - group: 939 +{% endif %} {% endif %} +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + {% endif %} \ No newline at end of file From ff29d9ca51501ce2df877ed0d2b30754f6ef22ff Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 11 Jul 2024 10:23:51 -0400 Subject: [PATCH 29/38] Update log-check to ignore kafka data directories Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/common/tools/sbin/so-log-check | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index cf1691589..2a86b9f2c 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -241,6 +241,7 @@ exclude_log "mysqld.log" # MySQL is removed as of 2.4.70, logs may still be on exclude_log "soctopus.log" # Soctopus is removed as of 2.4.70, logs may still be on disk exclude_log "agentstatus.log" # ignore this log since it tracks agents in error state exclude_log "detections_runtime-status_yara.log" # temporarily ignore this log until Detections is more stable +exclude_log "/nsm/kafka/data/" # ignore Kafka data directory from log check. for log_file in $(cat /tmp/log_check_files); do status "Checking log file $log_file" From 4182ff66a0368a181c79e12c145baa7e5e1c68e3 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 11 Jul 2024 16:37:16 -0400 Subject: [PATCH 30/38] rearrange kafka pillar, declutters SOC ui Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/kafka/config.map.jinja | 4 ++-- salt/kafka/ssl.sls | 2 +- salt/kafka/tools/sbin_jinja/so-kafka-trust | 2 +- .../pipelines/config/so/0800_input_kafka.conf.jinja | 4 ++-- salt/manager/tools/sbin/soup | 8 +++++++- setup/so-functions | 5 +++-- 6 files changed, 16 insertions(+), 9 deletions(-) diff --git a/salt/kafka/config.map.jinja b/salt/kafka/config.map.jinja index b054e0656..1e43a3ec7 100644 --- a/salt/kafka/config.map.jinja +++ b/salt/kafka/config.map.jinja @@ -6,8 +6,8 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% set KAFKA_NODES_PILLAR = salt['pillar.get']('kafka:nodes') %} -{% set KAFKA_PASSWORD = salt['pillar.get']('kafka:password') %} -{% set KAFKA_TRUSTPASS = salt['pillar.get']('kafka:trustpass') %} +{% set KAFKA_PASSWORD = salt['pillar.get']('kafka:config:password') %} +{% set KAFKA_TRUSTPASS = salt['pillar.get']('kafka:config:trustpass') %} {# Create list of KRaft controllers #} {% set controllers = [] %} diff --git a/salt/kafka/ssl.sls b/salt/kafka/ssl.sls index 50a01b22c..8604d4e7e 100644 --- a/salt/kafka/ssl.sls +++ b/salt/kafka/ssl.sls @@ -6,7 +6,7 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls.split('.')[0] in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} -{% set kafka_password = salt['pillar.get']('kafka:password') %} +{% set kafka_password = salt['pillar.get']('kafka:config:password') %} include: - ca.dirs diff --git a/salt/kafka/tools/sbin_jinja/so-kafka-trust b/salt/kafka/tools/sbin_jinja/so-kafka-trust index 8d404cb9a..55ba9612e 100644 --- a/salt/kafka/tools/sbin_jinja/so-kafka-trust +++ b/salt/kafka/tools/sbin_jinja/so-kafka-trust @@ -4,7 +4,7 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{% set TRUSTPASS = salt['pillar.get']('kafka:trustpass') %} +{% set TRUSTPASS = salt['pillar.get']('kafka:config:trustpass') %} if [ ! -f /opt/so/saltstack/local/salt/kafka/files/kafka-truststore ]; then docker run -v /etc/pki/ca.crt:/etc/pki/ca.crt --name so-kafkatrust --user root --entrypoint /opt/java/openjdk/bin/keytool {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kafka:{{ GLOBALS.so_version }} -import -file /etc/pki/ca.crt -alias SOS -keystore /etc/pki/kafka-truststore -storepass {{ TRUSTPASS }} -storetype jks -noprompt diff --git a/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja b/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja index 00dd6d530..7478375b0 100644 --- a/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja +++ b/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja @@ -1,5 +1,5 @@ -{%- set kafka_password = salt['pillar.get']('kafka:password') %} -{%- set kafka_trustpass = salt['pillar.get']('kafka:trustpass') %} +{%- set kafka_password = salt['pillar.get']('kafka:config:password') %} +{%- set kafka_trustpass = salt['pillar.get']('kafka:config:trustpass') %} {%- set kafka_brokers = salt['pillar.get']('kafka:nodes', {}) %} {%- set brokers = [] %} diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 019f29ebb..0ace4a5b2 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -674,7 +674,13 @@ up_to_2.4.80() { up_to_2.4.90() { kafkatrust=$(get_random_value) - echo ' trustpass: '$kafkatrust >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls + # rearranging the kafka pillar to reduce clutter in SOC UI + kafkasavedpass=$(so-yaml.py get /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.password) + kafkatrimpass=$(echo $kafkasavedpass | awk '{print $1}') + so-yaml.py remove /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.password + echo ' config:' >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls + echo ' password: '$kafkatrimpass >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls + echo ' trustpass: '$kafkatrust >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls INSTALLEDVERSION=2.4.90 diff --git a/setup/so-functions b/setup/so-functions index aa9eb1909..02467117e 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1187,8 +1187,9 @@ kafka_pillar() { printf '%s\n'\ "kafka:"\ " cluster_id: $KAFKACLUSTERID"\ - " password: $KAFKAPASS"\ - " trustpass: $KAFKATRUST" > $kafka_pillar_file + " config:"\ + " password: $KAFKAPASS"\ + " trustpass: $KAFKATRUST" > $kafka_pillar_file } logrotate_pillar() { From 7ea8d5efd01bd26cc91dbef41e4d287697a4b4ec Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 12 Jul 2024 14:44:10 -0400 Subject: [PATCH 31/38] Remove redis input pipeline from searchnodes when global pipeline is Kafka Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/logstash/map.jinja | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/logstash/map.jinja b/salt/logstash/map.jinja index 2c3e02864..8fc3291e5 100644 --- a/salt/logstash/map.jinja +++ b/salt/logstash/map.jinja @@ -33,6 +33,7 @@ {# Append Kafka input pipeline when Kafka is enabled #} {% if GLOBALS.pipeline == 'KAFKA' %} +{% do LOGSTASH_MERGED.defined_pipelines.search.remove('so/0900_input_redis.conf.jinja') %} {% do LOGSTASH_MERGED.defined_pipelines.search.append('so/0800_input_kafka.conf.jinja') %} {% do LOGSTASH_MERGED.defined_pipelines.manager.append('so/0800_input_kafka.conf.jinja') %} {# Disable logstash on manager & receiver nodes unless it has an override configured #} From 5d322ebc0b4a6d17521503c05c824aa574619848 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 12 Jul 2024 14:45:11 -0400 Subject: [PATCH 32/38] Allow searchnodes to run kafka.ssl state for kafka-logstash cert generation Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/kafka/ssl.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kafka/ssl.sls b/salt/kafka/ssl.sls index 8604d4e7e..04b6b4ba7 100644 --- a/salt/kafka/ssl.sls +++ b/salt/kafka/ssl.sls @@ -4,7 +4,7 @@ # Elastic License 2.0. {% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} +{% if sls.split('.')[0] in allowed_states or sls in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} {% set kafka_password = salt['pillar.get']('kafka:config:password') %} From fe1824aedd776bf9a286f67c41b6c68118538031 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 15 Jul 2024 11:28:59 -0400 Subject: [PATCH 33/38] Revert "Elastic 8.14.2" --- salt/common/tools/sbin/so-common | 2 +- .../elastic-defend-endpoints.json | 2 +- .../endpoints-initial/windows-defender.json | 2 +- .../tools/sbin_jinja/so-elastic-fleet-setup | 21 +- salt/elasticsearch/defaults.yaml | 2660 +++++++---------- salt/elasticsearch/soc_elasticsearch.yaml | 52 - .../so-elasticsearch-templates-load | 4 +- salt/kibana/files/config_saved_objects.ndjson | 2 +- .../tools/sbin_jinja/so-kibana-config-load | 2 +- 9 files changed, 1103 insertions(+), 1644 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 8a6effa5c..05c47a6c1 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -8,7 +8,7 @@ # Elastic agent is not managed by salt. Because of this we must store this base information in a # script that accompanies the soup system. Since so-common is one of those special soup files, # and since this same logic is required during installation, it's included in this file. -ELASTIC_AGENT_TARBALL_VERSION="8.14.2" +ELASTIC_AGENT_TARBALL_VERSION="8.10.4" ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5" ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" diff --git a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json index 15f08a151..de35f803b 100644 --- a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json +++ b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json @@ -5,7 +5,7 @@ "package": { "name": "endpoint", "title": "Elastic Defend", - "version": "8.14.0" + "version": "8.10.2" }, "enabled": true, "policy_id": "endpoints-initial", diff --git a/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json b/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json index ab7e0783f..ac4394e62 100644 --- a/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json +++ b/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json @@ -11,7 +11,7 @@ "winlogs-winlog": { "enabled": true, "streams": { - "winlog.winlogs": { + "winlog.winlog": { "enabled": true, "vars": { "channel": "Microsoft-Windows-Windows Defender/Operational", diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index 7e497f6f5..0748557fd 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -53,8 +53,7 @@ fi printf "\n### Create ES Token ###\n" ESTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/service_tokens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq -r .value) -### Create Outputs, Fleet Policy and Fleet URLs ### -# Create the Manager Elasticsearch Output first and set it as the default output +### Create Outputs & Fleet URLs ### printf "\nAdd Manager Elasticsearch Output...\n" ESCACRT=$(openssl x509 -in $INTCA) JSON_STRING=$( jq -n \ @@ -63,13 +62,7 @@ JSON_STRING=$( jq -n \ curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" printf "\n\n" -# Create the Manager Fleet Server Host Agent Policy -# This has to be done while the Elasticsearch Output is set to the default Output -printf "Create Manager Fleet Server Policy...\n" -elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "true" "120" - -# Now we can create the Logstash Output and set it to to be the default Output -printf "\n\nCreate Logstash Output Config if node is not an Import or Eval install\n" +printf "\nCreate Logstash Output Config if node is not an Import or Eval install\n" {% if grains.role not in ['so-import', 'so-eval'] %} LOGSTASHCRT=$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt) LOGSTASHKEY=$(openssl rsa -in /etc/pki/elasticfleet-logstash.key) @@ -108,6 +101,16 @@ printf "\n\n" # Load Elasticsearch templates /usr/sbin/so-elasticsearch-templates-load +# Manager Fleet Server Host +elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "true" "120" + +#Temp Fixup for ES Output bug +JSON_STRING=$( jq -n \ + --arg NAME "FleetServer_{{ GLOBALS.hostname }}" \ + '{"name": $NAME,"description": $NAME,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":120,"data_output_id":"so-manager_elasticsearch"}' + ) +curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/FleetServer_{{ GLOBALS.hostname }}" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" + # Initial Endpoints Policy elastic_fleet_policy_create "endpoints-initial" "Initial Endpoint Policy" "false" "1209600" diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index e1a2d192f..36d673d70 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -56,6 +56,87 @@ elasticsearch: enabled: true key: /usr/share/elasticsearch/config/elasticsearch.key verification_mode: none + pipelines: + custom001: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom001 + - pipeline: + name: common + custom002: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom002 + - pipeline: + name: common + custom003: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom003 + - pipeline: + name: common + custom004: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom004 + - pipeline: + name: common + custom005: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom005 + - pipeline: + name: common + custom006: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom006 + - pipeline: + name: common + custom007: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom007 + - pipeline: + name: common + custom008: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom008 + - pipeline: + name: common + custom009: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom009 + - pipeline: + name: common + custom010: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom010 + - pipeline: + name: common index_settings: global_overrides: index_template: @@ -89,13 +170,84 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-items: + index_sorting: false + index_template: + composed_of: + - so-items-mappings + index_patterns: + - .items-default-** + priority: 500 + template: + mappings: + date_detection: false + settings: + index: + lifecycle: + name: so-items-logs + rollover_alias: ".items-default" + routing: + allocation: + include: + _tier_preference: "data_content" + mapping: + total_fields: + limit: 10000 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc + policy: + phases: + hot: + actions: + rollover: + max_size: 50gb + min_age: 0ms + so-lists: + index_sorting: false + index_template: + composed_of: + - so-lists-mappings + index_patterns: + - .lists-default-** + priority: 500 + template: + mappings: + date_detection: false + settings: + index: + lifecycle: + name: so-lists-logs + rollover_alias: ".lists-default" + routing: + allocation: + include: + _tier_preference: "data_content" + mapping: + total_fields: + limit: 10000 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc + policy: + phases: + hot: + actions: + rollover: + max_size: 50gb + min_age: 0ms so-case: index_sorting: false index_template: composed_of: - case-mappings - case-settings - ignore_missing_component_templates: [] index_patterns: - so-case* priority: 500 @@ -119,6 +271,142 @@ elasticsearch: sort: field: '@timestamp' order: desc + so-detection: + index_sorting: false + index_template: + composed_of: + - detection-mappings + - detection-settings + index_patterns: + - so-detection* + priority: 500 + template: + mappings: + date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string + settings: + index: + mapping: + total_fields: + limit: 1500 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc + so-logs-soc: + close: 30 + delete: 365 + index_sorting: false + index_template: + composed_of: + - agent-mappings + - dtc-agent-mappings + - base-mappings + - dtc-base-mappings + - client-mappings + - dtc-client-mappings + - container-mappings + - destination-mappings + - dtc-destination-mappings + - pb-override-destination-mappings + - dll-mappings + - dns-mappings + - dtc-dns-mappings + - ecs-mappings + - dtc-ecs-mappings + - error-mappings + - event-mappings + - dtc-event-mappings + - file-mappings + - dtc-file-mappings + - group-mappings + - host-mappings + - dtc-host-mappings + - http-mappings + - dtc-http-mappings + - log-mappings + - network-mappings + - dtc-network-mappings + - observer-mappings + - dtc-observer-mappings + - organization-mappings + - package-mappings + - process-mappings + - dtc-process-mappings + - related-mappings + - rule-mappings + - dtc-rule-mappings + - server-mappings + - service-mappings + - dtc-service-mappings + - source-mappings + - dtc-source-mappings + - pb-override-source-mappings + - threat-mappings + - tls-mappings + - url-mappings + - user_agent-mappings + - dtc-user_agent-mappings + - common-settings + - common-dynamic-mappings + data_stream: {} + index_patterns: + - logs-soc-so* + priority: 500 + template: + mappings: + date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string + settings: + index: + lifecycle: + name: so-soc-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + warm: 7 so-common: close: 30 delete: 365 @@ -185,7 +473,6 @@ elasticsearch: - common-dynamic-mappings - winlog-mappings data_stream: {} - ignore_missing_component_templates: [] index_patterns: - logs-*-so* priority: 1 @@ -236,36 +523,6 @@ elasticsearch: priority: 50 min_age: 30d warm: 7 - so-detection: - index_sorting: false - index_template: - composed_of: - - detection-mappings - - detection-settings - ignore_missing_component_templates: [] - index_patterns: - - so-detection* - priority: 500 - template: - mappings: - date_detection: false - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - settings: - index: - mapping: - total_fields: - limit: 1500 - number_of_replicas: 0 - number_of_shards: 1 - refresh_interval: 30s - sort: - field: '@timestamp' - order: desc so-endgame: index_sorting: false index_template: @@ -328,7 +585,6 @@ elasticsearch: - common-settings - common-dynamic-mappings - winlog-mappings - ignore_missing_component_templates: [] index_patterns: - endgame* priority: 500 @@ -434,7 +690,6 @@ elasticsearch: - dtc-user_agent-mappings - common-settings - common-dynamic-mappings - ignore_missing_component_templates: [] index_patterns: - so-idh-* priority: 500 @@ -547,7 +802,6 @@ elasticsearch: - common-dynamic-mappings - winlog-mappings data_stream: {} - ignore_missing_component_templates: [] index_patterns: - logs-import-so* priority: 500 @@ -598,91 +852,6 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-items: - index_sorting: false - index_template: - composed_of: - - so-items-mappings - ignore_missing_component_templates: [] - index_patterns: - - .items-default-** - priority: 500 - template: - mappings: - date_detection: false - settings: - index: - lifecycle: - name: so-items-logs - rollover_alias: .items-default - mapping: - total_fields: - limit: 10000 - number_of_replicas: 0 - number_of_shards: 1 - refresh_interval: 30s - routing: - allocation: - include: - _tier_preference: data_content - sort: - field: '@timestamp' - order: desc - policy: - phases: - hot: - actions: - rollover: - max_size: 50gb - min_age: 0ms - so-kismet: - index_sorting: false - index_template: - composed_of: - - kismet-mappings - - source-mappings - - client-mappings - - device-mappings - - network-mappings - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: [] - index_patterns: - - logs-kismet-so* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-kismet-logs - number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d so-kratos: close: 30 delete: 365 @@ -742,7 +911,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: [] index_patterns: - logs-kratos-so* priority: 500 @@ -793,43 +961,6 @@ elasticsearch: priority: 50 min_age: 30d warm: 7 - so-lists: - index_sorting: false - index_template: - composed_of: - - so-lists-mappings - ignore_missing_component_templates: [] - index_patterns: - - .lists-default-** - priority: 500 - template: - mappings: - date_detection: false - settings: - index: - lifecycle: - name: so-lists-logs - rollover_alias: .lists-default - mapping: - total_fields: - limit: 10000 - number_of_replicas: 0 - number_of_shards: 1 - refresh_interval: 30s - routing: - allocation: - include: - _tier_preference: data_content - sort: - field: '@timestamp' - order: desc - policy: - phases: - hot: - actions: - rollover: - max_size: 50gb - min_age: 0ms so-logs: index_sorting: false index_template: @@ -842,7 +973,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: [] index_patterns: - logs-*-* priority: 225 @@ -904,8 +1034,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-1password.item_usages@custom index_patterns: - logs-1password.item_usages-* priority: 501 @@ -950,8 +1078,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-1password.signin_attempts@custom index_patterns: - logs-1password.signin_attempts-* priority: 501 @@ -996,8 +1122,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-apache.access@custom index_patterns: - logs-apache.access-* priority: 501 @@ -1042,8 +1166,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-apache.error@custom index_patterns: - logs-apache.error-* priority: 501 @@ -1088,8 +1210,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-auditd.log@custom index_patterns: - logs-auditd.log-* priority: 501 @@ -1134,8 +1254,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-auth0.logs@custom index_patterns: - logs-auth0.logs-* priority: 501 @@ -1170,27 +1288,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-aws_x_cloudfront_logs: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-aws.cloudfront_logs@package - - logs-aws.cloudfront_logs@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-aws.cloudfront_logs@custom index_patterns: - - logs-aws.cloudfront_logs-* - priority: 501 + - "logs-aws.cloudfront_logs-*" template: settings: index: lifecycle: name: so-logs-aws.cloudfront_logs-logs number_of_replicas: 0 + composed_of: + - "logs-aws.cloudfront_logs@package" + - "logs-aws.cloudfront_logs@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -1226,8 +1342,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-aws.cloudtrail@custom index_patterns: - logs-aws.cloudtrail-* priority: 501 @@ -1272,8 +1386,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-aws.cloudwatch_logs@custom index_patterns: - logs-aws.cloudwatch_logs-* priority: 501 @@ -1318,8 +1430,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-aws.ec2_logs@custom index_patterns: - logs-aws.ec2_logs-* priority: 501 @@ -1364,8 +1474,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-aws.elb_logs@custom index_patterns: - logs-aws.elb_logs-* priority: 501 @@ -1410,8 +1518,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-aws.firewall_logs@custom index_patterns: - logs-aws.firewall_logs-* priority: 501 @@ -1446,27 +1552,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-aws_x_guardduty: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-aws.guardduty@package - - logs-aws.guardduty@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-aws.guardduty@custom index_patterns: - - logs-aws.guardduty-* - priority: 501 + - "logs-aws.guardduty-*" template: settings: index: lifecycle: name: so-logs-aws.guardduty-logs number_of_replicas: 0 + composed_of: + - "logs-aws.guardduty@package" + - "logs-aws.guardduty@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -1492,27 +1596,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-aws_x_inspector: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-aws.inspector@package - - logs-aws.inspector@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-aws.inspector@custom index_patterns: - - logs-aws.inspector-* - priority: 501 + - "logs-aws.inspector-*" template: settings: index: lifecycle: name: so-logs-aws.inspector-logs number_of_replicas: 0 + composed_of: + - "logs-aws.inspector@package" + - "logs-aws.inspector@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -1548,8 +1650,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-aws.route53_public_logs@custom index_patterns: - logs-aws.route53_public_logs-* priority: 501 @@ -1594,8 +1694,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-aws.route53_resolver_logs@custom index_patterns: - logs-aws.route53_resolver_logs-* priority: 501 @@ -1640,8 +1738,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-aws.s3access@custom index_patterns: - logs-aws.s3access-* priority: 501 @@ -1676,27 +1772,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-aws_x_securityhub_findings: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-aws.securityhub_findings@package - - logs-aws.securityhub_findings@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-aws.securityhub_findings@custom index_patterns: - - logs-aws.securityhub_findings-* - priority: 501 + - "logs-aws.securityhub_findings-*" template: settings: index: lifecycle: name: so-logs-aws.securityhub_findings-logs number_of_replicas: 0 + composed_of: + - "logs-aws.securityhub_findings@package" + - "logs-aws.securityhub_findings@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -1722,27 +1816,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-aws_x_securityhub_insights: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-aws.securityhub_insights@package - - logs-aws.securityhub_insights@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-aws.securityhub_insights@custom index_patterns: - - logs-aws.securityhub_insights-* - priority: 501 + - "logs-aws.securityhub_insights-*" template: settings: index: lifecycle: name: so-logs-aws.securityhub_insights-logs number_of_replicas: 0 + composed_of: + - "logs-aws.securityhub_insights@package" + - "logs-aws.securityhub_insights@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -1778,8 +1870,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-aws.vpcflow@custom index_patterns: - logs-aws.vpcflow-* priority: 501 @@ -1824,8 +1914,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-aws.waf@custom index_patterns: - logs-aws.waf-* priority: 501 @@ -1870,8 +1958,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-azure.activitylogs@custom index_patterns: - logs-azure.activitylogs-* priority: 501 @@ -1916,8 +2002,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-azure.application_gateway@custom index_patterns: - logs-azure.application_gateway-* priority: 501 @@ -1962,8 +2046,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-azure.auditlogs@custom index_patterns: - logs-azure.auditlogs-* priority: 501 @@ -2008,8 +2090,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-azure.eventhub@custom index_patterns: - logs-azure.eventhub-* priority: 501 @@ -2054,8 +2134,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-azure.firewall_logs@custom index_patterns: - logs-azure.firewall_logs-* priority: 501 @@ -2100,8 +2178,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-azure.identity_protection@custom index_patterns: - logs-azure.identity_protection-* priority: 501 @@ -2146,8 +2222,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-azure.platformlogs@custom index_patterns: - logs-azure.platformlogs-* priority: 501 @@ -2192,8 +2266,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-azure.provisioning@custom index_patterns: - logs-azure.provisioning-* priority: 501 @@ -2238,8 +2310,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-azure.signinlogs@custom index_patterns: - logs-azure.signinlogs-* priority: 501 @@ -2284,8 +2354,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-azure.springcloudlogs@custom index_patterns: - logs-azure.springcloudlogs-* priority: 501 @@ -2330,8 +2398,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-barracuda.waf@custom index_patterns: - logs-barracuda.waf-* priority: 501 @@ -2376,8 +2442,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-carbonblack_edr.log@custom index_patterns: - logs-carbonblack_edr.log-* priority: 501 @@ -2412,27 +2476,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-cef_x_log: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-cef.log@package - - logs-cef.log@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-cef.log@custom index_patterns: - - logs-cef.log-* - priority: 501 + - "logs-cef.log-*" template: settings: index: lifecycle: name: so-logs-cef.log-logs number_of_replicas: 0 + composed_of: + - "logs-cef.log@package" + - "logs-cef.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -2458,27 +2520,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-checkpoint_x_firewall: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-checkpoint.firewall@package - - logs-checkpoint.firewall@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-checkpoint.firewall@custom index_patterns: - - logs-checkpoint.firewall-* - priority: 501 + - "logs-checkpoint.firewall-*" template: settings: index: lifecycle: name: so-logs-checkpoint.firewall-logs number_of_replicas: 0 + composed_of: + - "logs-checkpoint.firewall@package" + - "logs-checkpoint.firewall@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -2514,8 +2574,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-cisco_asa.log@custom index_patterns: - logs-cisco_asa.log-* priority: 501 @@ -2560,8 +2618,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-cisco_duo.admin@custom index_patterns: - logs-cisco_duo.admin-* priority: 501 @@ -2606,8 +2662,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-cisco_duo.auth@custom index_patterns: - logs-cisco_duo.auth-* priority: 501 @@ -2652,8 +2706,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-cisco_duo.offline_enrollment@custom index_patterns: - logs-cisco_duo.offline_enrollment-* priority: 501 @@ -2698,8 +2750,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-cisco_duo.summary@custom index_patterns: - logs-cisco_duo.summary-* priority: 501 @@ -2744,8 +2794,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-cisco_duo.telephony@custom index_patterns: - logs-cisco_duo.telephony-* priority: 501 @@ -2780,27 +2828,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-cisco_ftd_x_log: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-cisco_ftd.log@package - - logs-cisco_ftd.log@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-cisco_ftd.log@custom index_patterns: - - logs-cisco_ftd.log-* - priority: 501 + - "logs-cisco_ftd.log-*" template: settings: index: lifecycle: name: so-logs-cisco_ftd.log-logs number_of_replicas: 0 + composed_of: + - "logs-cisco_ftd.log@package" + - "logs-cisco_ftd.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -2826,27 +2872,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-cisco_ios_x_log: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-cisco_ios.log@package - - logs-cisco_ios.log@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-cisco_ios.log@custom index_patterns: - - logs-cisco_ios.log-* - priority: 501 + - "logs-cisco_ios.log-*" template: settings: index: lifecycle: name: so-logs-cisco_ios.log-logs number_of_replicas: 0 + composed_of: + - "logs-cisco_ios.log@package" + - "logs-cisco_ios.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -2872,27 +2916,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-cisco_ise_x_log: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-cisco_ise.log@package - - logs-cisco_ise.log@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-cisco_ise.log@custom index_patterns: - - logs-cisco_ise.log-* - priority: 501 + - "logs-cisco_ise.log-*" template: settings: index: lifecycle: name: so-logs-cisco_ise.log-logs number_of_replicas: 0 + composed_of: + - "logs-cisco_ise.log@package" + - "logs-cisco_ise.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -2928,8 +2970,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-cisco_meraki.events@custom index_patterns: - logs-cisco_meraki.events-* priority: 501 @@ -2974,8 +3014,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-cisco_meraki.log@custom index_patterns: - logs-cisco_meraki.log-* priority: 501 @@ -3020,8 +3058,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-cisco_umbrella.log@custom index_patterns: - logs-cisco_umbrella.log-* priority: 501 @@ -3056,27 +3092,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_adc_x_interface: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-citrix_adc.interface@package - - logs-citrix_adc.interface@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-citrix_adc.interface@custom index_patterns: - - logs-citrix_adc.interface-* - priority: 501 + - "logs-citrix_adc.interface-*" template: settings: index: lifecycle: name: so-logs-citrix_adc.interface-logs number_of_replicas: 0 + composed_of: + - "logs-citrix_adc.interface@package" + - "logs-citrix_adc.interface@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -3102,27 +3136,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_adc_x_lbvserver: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-citrix_adc.lbvserver@package - - logs-citrix_adc.lbvserver@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-citrix_adc.lbvserver@custom index_patterns: - - logs-citrix_adc.lbvserver-* - priority: 501 + - "logs-citrix_adc.lbvserver-*" template: settings: index: lifecycle: name: so-logs-citrix_adc.lbvserver-logs number_of_replicas: 0 + composed_of: + - "logs-citrix_adc.lbvserver@package" + - "logs-citrix_adc.lbvserver@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -3148,27 +3180,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_adc_x_service: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-citrix_adc.service@package - - logs-citrix_adc.service@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-citrix_adc.service@custom index_patterns: - - logs-citrix_adc.service-* - priority: 501 + - "logs-citrix_adc.service-*" template: settings: index: lifecycle: name: so-logs-citrix_adc.service-logs number_of_replicas: 0 + composed_of: + - "logs-citrix_adc.service@package" + - "logs-citrix_adc.service@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -3194,27 +3224,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_adc_x_system: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-citrix_adc.system@package - - logs-citrix_adc.system@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-citrix_adc.system@custom index_patterns: - - logs-citrix_adc.system-* - priority: 501 + - "logs-citrix_adc.system-*" template: settings: index: lifecycle: name: so-logs-citrix_adc.system-logs number_of_replicas: 0 + composed_of: + - "logs-citrix_adc.system@package" + - "logs-citrix_adc.system@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -3240,27 +3268,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_adc_x_vpn: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-citrix_adc.vpn@package - - logs-citrix_adc.vpn@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-citrix_adc.vpn@custom index_patterns: - - logs-citrix_adc.vpn-* - priority: 501 + - "logs-citrix_adc.vpn-*" template: settings: index: lifecycle: name: so-logs-citrix_adc.vpn-logs number_of_replicas: 0 + composed_of: + - "logs-citrix_adc.vpn@package" + - "logs-citrix_adc.vpn@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -3286,27 +3312,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_waf_x_log: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-citrix_waf.log@package - - logs-citrix_waf.log@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-citrix_waf.log@custom index_patterns: - - logs-citrix_waf.log-* - priority: 501 + - "logs-citrix_waf.log-*" template: settings: index: lifecycle: name: so-logs-citrix_waf.log-logs number_of_replicas: 0 + composed_of: + - "logs-citrix_waf.log@package" + - "logs-citrix_waf.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -3342,8 +3366,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-cloudflare.audit@custom index_patterns: - logs-cloudflare.audit-* priority: 501 @@ -3388,8 +3410,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-cloudflare.logpull@custom index_patterns: - logs-cloudflare.logpull-* priority: 501 @@ -3434,8 +3454,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-crowdstrike.falcon@custom index_patterns: - logs-crowdstrike.falcon-* priority: 501 @@ -3480,8 +3498,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-crowdstrike.fdr@custom index_patterns: - logs-crowdstrike.fdr-* priority: 501 @@ -3526,8 +3542,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-darktrace.ai_analyst_alert@custom index_patterns: - logs-darktrace.ai_analyst_alert-* priority: 501 @@ -3572,8 +3586,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-darktrace.model_breach_alert@custom index_patterns: - logs-darktrace.model_breach_alert-* priority: 501 @@ -3618,8 +3630,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-darktrace.system_status_alert@custom index_patterns: - logs-darktrace.system_status_alert-* priority: 501 @@ -3665,7 +3675,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: [] index_patterns: - logs-detections.alerts-* priority: 501 @@ -3728,8 +3737,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-elastic_agent@custom index_patterns: - logs-elastic_agent-* priority: 501 @@ -3791,8 +3798,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-elastic_agent.apm_server@custom index_patterns: - logs-elastic_agent.apm_server-* priority: 501 @@ -3854,8 +3859,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-elastic_agent.auditbeat@custom index_patterns: - logs-elastic_agent.auditbeat-* priority: 501 @@ -3914,8 +3917,6 @@ elasticsearch: - logs-elastic_agent.cloudbeat@custom - so-fleet_globals-1 - so-fleet_agent_id_verification-1 - ignore_missing_component_templates: - - logs-elastic_agent.cloudbeat@custom index_patterns: - logs-elastic_agent.cloudbeat-* priority: 501 @@ -3978,8 +3979,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-elastic_agent.endpoint_security@custom index_patterns: - logs-elastic_agent.endpoint_security-* priority: 501 @@ -4036,8 +4035,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-elastic_agent.filebeat@custom index_patterns: - logs-elastic_agent.filebeat-* priority: 501 @@ -4094,8 +4091,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-elastic_agent.fleet_server@custom index_patterns: - logs-elastic_agent.fleet_server-* priority: 501 @@ -4145,8 +4140,6 @@ elasticsearch: - logs-elastic_agent.heartbeat@custom - so-fleet_globals-1 - so-fleet_agent_id_verification-1 - ignore_missing_component_templates: - - logs-elastic_agent.heartbeat@custom index_patterns: - logs-elastic_agent.heartbeat-* priority: 501 @@ -4209,8 +4202,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-elastic_agent.metricbeat@custom index_patterns: - logs-elastic_agent.metricbeat-* priority: 501 @@ -4267,8 +4258,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-elastic_agent.osquerybeat@custom index_patterns: - logs-elastic_agent.osquerybeat-* priority: 501 @@ -4324,8 +4313,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-elastic_agent.packetbeat@custom index_patterns: - logs-elastic_agent.packetbeat-* priority: 501 @@ -4388,8 +4375,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-endpoint.alerts@custom index_patterns: - logs-endpoint.alerts-* priority: 501 @@ -4446,8 +4431,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-endpoint.diagnostic.collection@custom index_patterns: - .logs-endpoint.diagnostic.collection-* priority: 501 @@ -4504,8 +4487,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-endpoint.events.api@custom index_patterns: - logs-endpoint.events.api-* priority: 501 @@ -4562,8 +4543,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-endpoint.events.file@custom index_patterns: - logs-endpoint.events.file-* priority: 501 @@ -4620,8 +4599,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-endpoint.events.library@custom index_patterns: - logs-endpoint.events.library-* priority: 501 @@ -4678,8 +4655,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-endpoint.events.network@custom index_patterns: - logs-endpoint.events.network-* priority: 501 @@ -4736,8 +4711,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-endpoint.events.process@custom index_patterns: - logs-endpoint.events.process-* priority: 501 @@ -4794,8 +4767,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-endpoint.events.registry@custom index_patterns: - logs-endpoint.events.registry-* priority: 501 @@ -4852,8 +4823,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-endpoint.events.security@custom index_patterns: - logs-endpoint.events.security-* priority: 501 @@ -4909,8 +4878,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-f5_bigip.log@custom index_patterns: - logs-f5_bigip.log-* priority: 501 @@ -4955,8 +4922,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-fim.event@custom index_patterns: - logs-fim.event-* priority: 501 @@ -5001,8 +4966,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-fireeye.nx@custom index_patterns: - logs-fireeye.nx-* priority: 501 @@ -5047,8 +5010,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-fortinet_fortigate.log@custom index_patterns: - logs-fortinet_fortigate.log-* priority: 501 @@ -5093,8 +5054,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-fortinet.clientendpoint@custom index_patterns: - logs-fortinet.clientendpoint-* priority: 501 @@ -5139,8 +5098,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-fortinet.firewall@custom index_patterns: - logs-fortinet.firewall-* priority: 501 @@ -5185,8 +5142,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-fortinet.fortimail@custom index_patterns: - logs-fortinet.fortimail-* priority: 501 @@ -5231,8 +5186,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-fortinet.fortimanager@custom index_patterns: - logs-fortinet.fortimanager-* priority: 501 @@ -5277,8 +5230,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-gcp.audit@custom index_patterns: - logs-gcp.audit-* priority: 501 @@ -5323,8 +5274,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-gcp.dns@custom index_patterns: - logs-gcp.dns-* priority: 501 @@ -5369,8 +5318,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-gcp.firewall@custom index_patterns: - logs-gcp.firewall-* priority: 501 @@ -5415,8 +5362,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-gcp.loadbalancing_logs@custom index_patterns: - logs-gcp.loadbalancing_logs-* priority: 501 @@ -5461,8 +5406,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-gcp.vpcflow@custom index_patterns: - logs-gcp.vpcflow-* priority: 501 @@ -5507,8 +5450,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-github.audit@custom index_patterns: - logs-github.audit-* priority: 501 @@ -5553,8 +5494,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-github.code_scanning@custom index_patterns: - logs-github.code_scanning-* priority: 501 @@ -5599,8 +5538,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-github.dependabot@custom index_patterns: - logs-github.dependabot-* priority: 501 @@ -5645,8 +5582,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-github.issues@custom index_patterns: - logs-github.issues-* priority: 501 @@ -5691,8 +5626,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-github.secret_scanning@custom index_patterns: - logs-github.secret_scanning-* priority: 501 @@ -5737,8 +5670,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.access_transparency@custom index_patterns: - logs-google_workspace.access_transparency-* priority: 501 @@ -5783,8 +5714,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.admin@custom index_patterns: - logs-google_workspace.admin-* priority: 501 @@ -5829,8 +5758,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.alert@custom index_patterns: - logs-google_workspace.alert-* priority: 501 @@ -5875,8 +5802,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.context_aware_access@custom index_patterns: - logs-google_workspace.context_aware_access-* priority: 501 @@ -5921,8 +5846,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.device@custom index_patterns: - logs-google_workspace.device-* priority: 501 @@ -5967,8 +5890,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.drive@custom index_patterns: - logs-google_workspace.drive-* priority: 501 @@ -6013,8 +5934,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.gcp@custom index_patterns: - logs-google_workspace.gcp-* priority: 501 @@ -6059,8 +5978,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.group_enterprise@custom index_patterns: - logs-google_workspace.group_enterprise-* priority: 501 @@ -6105,8 +6022,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.groups@custom index_patterns: - logs-google_workspace.groups-* priority: 501 @@ -6151,8 +6066,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.login@custom index_patterns: - logs-google_workspace.login-* priority: 501 @@ -6197,8 +6110,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.rules@custom index_patterns: - logs-google_workspace.rules-* priority: 501 @@ -6243,8 +6154,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.saml@custom index_patterns: - logs-google_workspace.saml-* priority: 501 @@ -6289,8 +6198,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.token@custom index_patterns: - logs-google_workspace.token-* priority: 501 @@ -6335,8 +6242,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.user_accounts@custom index_patterns: - logs-google_workspace.user_accounts-* priority: 501 @@ -6381,9 +6286,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-http_endpoint.generic@package - - logs-http_endpoint.generic@custom index_patterns: - logs-http_endpoint.generic-* priority: 501 @@ -6428,8 +6330,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-httpjson.generic@custom index_patterns: - logs-httpjson.generic-* priority: 501 @@ -6464,27 +6364,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-iis_x_access: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-iis.access@package - - logs-iis.access@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-iis.access@custom index_patterns: - - logs-iis.access-* - priority: 501 + - "logs-iis.access-*" template: settings: index: lifecycle: name: so-logs-iis.access-logs number_of_replicas: 0 + composed_of: + - "logs-iis.access@package" + - "logs-iis.access@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -6510,27 +6408,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-iis_x_error: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-iis.error@package - - logs-iis.error@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-iis.error@custom index_patterns: - - logs-iis.error-* - priority: 501 + - "logs-iis.error-*" template: settings: index: lifecycle: name: so-logs-iis.error-logs number_of_replicas: 0 + composed_of: + - "logs-iis.error@package" + - "logs-iis.error@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -6566,8 +6462,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-juniper_srx.log@custom index_patterns: - logs-juniper_srx.log-* priority: 501 @@ -6612,8 +6506,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-juniper.junos@custom index_patterns: - logs-juniper.junos-* priority: 501 @@ -6658,8 +6550,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-juniper.netscreen@custom index_patterns: - logs-juniper.netscreen-* priority: 501 @@ -6704,8 +6594,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-juniper.srx@custom index_patterns: - logs-juniper.srx-* priority: 501 @@ -6750,8 +6638,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-kafka_log.generic@custom index_patterns: - logs-kafka_log.generic-* priority: 501 @@ -6796,8 +6682,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-lastpass.detailed_shared_folder@custom index_patterns: - logs-lastpass.detailed_shared_folder-* priority: 501 @@ -6842,8 +6726,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-lastpass.event_report@custom index_patterns: - logs-lastpass.event_report-* priority: 501 @@ -6888,8 +6770,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-lastpass.user@custom index_patterns: - logs-lastpass.user-* priority: 501 @@ -6934,8 +6814,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-m365_defender.event@custom index_patterns: - logs-m365_defender.event-* priority: 501 @@ -6980,8 +6858,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-m365_defender.incident@custom index_patterns: - logs-m365_defender.incident-* priority: 501 @@ -7026,8 +6902,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-m365_defender.log@custom index_patterns: - logs-m365_defender.log-* priority: 501 @@ -7072,8 +6946,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-microsoft_defender_endpoint.log@custom index_patterns: - logs-microsoft_defender_endpoint.log-* priority: 501 @@ -7118,8 +6990,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-microsoft_dhcp.log@custom index_patterns: - logs-microsoft_dhcp.log-* priority: 501 @@ -7154,27 +7024,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-microsoft_sqlserver_x_audit: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-microsoft_sqlserver.audit@package - - logs-microsoft_sqlserver.audit@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-microsoft_sqlserver.audit@custom index_patterns: - - logs-microsoft_sqlserver.audit-* - priority: 501 + - "logs-microsoft_sqlserver.audit-*" template: settings: index: lifecycle: name: so-logs-microsoft_sqlserver.audit-logs number_of_replicas: 0 + composed_of: + - "logs-microsoft_sqlserver.audit@package" + - "logs-microsoft_sqlserver.audit@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -7200,27 +7068,113 @@ elasticsearch: priority: 50 min_age: 30d so-logs-microsoft_sqlserver_x_log: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-microsoft_sqlserver.log@package - - logs-microsoft_sqlserver.log@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-microsoft_sqlserver.log@custom index_patterns: - - logs-microsoft_sqlserver.log-* - priority: 501 + - "logs-microsoft_sqlserver.log-*" template: settings: index: lifecycle: name: so-logs-microsoft_sqlserver.log-logs number_of_replicas: 0 + composed_of: + - "logs-microsoft_sqlserver.log@package" + - "logs-microsoft_sqlserver.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-mysql_x_error: + index_sorting: False + index_template: + index_patterns: + - "logs-mysql.error-*" + template: + settings: + index: + lifecycle: + name: so-logs-mysql.error-logs + number_of_replicas: 0 + composed_of: + - "logs-mysql.error@package" + - "logs-mysql.error@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-mysql_x_slowlog: + index_sorting: False + index_template: + index_patterns: + - "logs-mysql.slowlog-*" + template: + settings: + index: + lifecycle: + name: so-logs-mysql.slowlog-logs + number_of_replicas: 0 + composed_of: + - "logs-mysql.slowlog@package" + - "logs-mysql.slowlog@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -7256,8 +7210,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-mimecast.audit_events@custom index_patterns: - logs-mimecast.audit_events-* priority: 501 @@ -7302,8 +7254,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-mimecast.dlp_logs@custom index_patterns: - logs-mimecast.dlp_logs-* priority: 501 @@ -7348,8 +7298,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-mimecast.siem_logs@custom index_patterns: - logs-mimecast.siem_logs-* priority: 501 @@ -7394,8 +7342,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-mimecast.threat_intel_malware_customer@custom index_patterns: - logs-mimecast.threat_intel_malware_customer-* priority: 501 @@ -7440,8 +7386,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-mimecast.threat_intel_malware_grid@custom index_patterns: - logs-mimecast.threat_intel_malware_grid-* priority: 501 @@ -7486,8 +7430,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-mimecast.ttp_ap_logs@custom index_patterns: - logs-mimecast.ttp_ap_logs-* priority: 501 @@ -7532,8 +7474,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-mimecast.ttp_ip_logs@custom index_patterns: - logs-mimecast.ttp_ip_logs-* priority: 501 @@ -7578,8 +7518,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-mimecast.ttp_url_logs@custom index_patterns: - logs-mimecast.ttp_url_logs-* priority: 501 @@ -7613,98 +7551,6 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-logs-mysql_x_error: - index_sorting: false - index_template: - composed_of: - - logs-mysql.error@package - - logs-mysql.error@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-mysql.error@custom - index_patterns: - - logs-mysql.error-* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-logs-mysql.error-logs - number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-logs-mysql_x_slowlog: - index_sorting: false - index_template: - composed_of: - - logs-mysql.slowlog@package - - logs-mysql.slowlog@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-mysql.slowlog@custom - index_patterns: - - logs-mysql.slowlog-* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-logs-mysql.slowlog-logs - number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d so-logs-netflow_x_log: index_sorting: false index_template: @@ -7716,8 +7562,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-netflow.log@custom index_patterns: - logs-netflow.log-* priority: 501 @@ -7752,27 +7596,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-nginx_x_access: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-nginx.access@package - - logs-nginx.access@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-nginx.access@custom index_patterns: - - logs-nginx.access-* - priority: 501 + - "logs-nginx.access-*" template: settings: index: lifecycle: name: so-logs-nginx.access-logs number_of_replicas: 0 + composed_of: + - "logs-nginx.access@package" + - "logs-nginx.access@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -7798,27 +7640,69 @@ elasticsearch: priority: 50 min_age: 30d so-logs-nginx_x_error: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-nginx.error@package - - logs-nginx.error@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-nginx.error@custom index_patterns: - - logs-nginx.error-* - priority: 501 + - "logs-nginx.error-*" template: settings: index: lifecycle: name: so-logs-nginx.error-logs number_of_replicas: 0 + composed_of: + - "logs-nginx.error@package" + - "logs-nginx.error@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-nginx_x_stubstatus: + index_sorting: False + index_template: + index_patterns: + - "metrics-nginx.stubstatus-*" + template: + settings: + index: + lifecycle: + name: so-metrics-nginx.stubstatus-logs + number_of_replicas: 0 + composed_of: + - "metrics-nginx.stubstatus@package" + - "metrics-nginx.stubstatus@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -7854,8 +7738,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-o365.audit@custom index_patterns: - logs-o365.audit-* priority: 501 @@ -7900,8 +7782,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-okta.system@custom index_patterns: - logs-okta.system-* priority: 501 @@ -7945,7 +7825,6 @@ elasticsearch: name: elastic_agent composed_of: - logs-osquery_manager.action.responses - ignore_missing_component_templates: [] index_patterns: - .logs-osquery_manager.action.responses* priority: 501 @@ -7963,7 +7842,6 @@ elasticsearch: name: elastic_agent composed_of: - logs-osquery_manager.actions - ignore_missing_component_templates: [] index_patterns: - .logs-osquery_manager.actions* priority: 501 @@ -7982,8 +7860,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-panw.panos@custom index_patterns: - logs-panw.panos-* priority: 501 @@ -8028,8 +7904,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-pfsense.log@custom index_patterns: - logs-pfsense.log-* priority: 501 @@ -8064,27 +7938,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-proofpoint_tap_x_clicks_blocked: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-proofpoint_tap.clicks_blocked@package - - logs-proofpoint_tap.clicks_blocked@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-proofpoint_tap.clicks_blocked@custom index_patterns: - - logs-proofpoint_tap.clicks_blocked-* - priority: 501 + - "logs-proofpoint_tap.clicks_blocked-*" template: settings: index: lifecycle: name: so-logs-proofpoint_tap.clicks_blocked-logs number_of_replicas: 0 + composed_of: + - "logs-proofpoint_tap.clicks_blocked@package" + - "logs-proofpoint_tap.clicks_blocked@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -8110,27 +7982,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-proofpoint_tap_x_clicks_permitted: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-proofpoint_tap.clicks_permitted@package - - logs-proofpoint_tap.clicks_permitted@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-proofpoint_tap.clicks_permitted@custom index_patterns: - - logs-proofpoint_tap.clicks_permitted-* - priority: 501 + - "logs-proofpoint_tap.clicks_permitted-*" template: settings: index: lifecycle: name: so-logs-proofpoint_tap.clicks_permitted-logs number_of_replicas: 0 + composed_of: + - "logs-proofpoint_tap.clicks_permitted@package" + - "logs-proofpoint_tap.clicks_permitted@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -8156,27 +8026,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-proofpoint_tap_x_message_blocked: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-proofpoint_tap.message_blocked@package - - logs-proofpoint_tap.message_blocked@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-proofpoint_tap.message_blocked@custom index_patterns: - - logs-proofpoint_tap.message_blocked-* - priority: 501 + - "logs-proofpoint_tap.message_blocked-*" template: settings: index: lifecycle: name: so-logs-proofpoint_tap.message_blocked-logs number_of_replicas: 0 + composed_of: + - "logs-proofpoint_tap.message_blocked@package" + - "logs-proofpoint_tap.message_blocked@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -8202,27 +8070,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-proofpoint_tap_x_message_delivered: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-proofpoint_tap.message_delivered@package - - logs-proofpoint_tap.message_delivered@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-proofpoint_tap.message_delivered@custom index_patterns: - - logs-proofpoint_tap.message_delivered-* - priority: 501 + - "logs-proofpoint_tap.message_delivered-*" template: settings: index: lifecycle: name: so-logs-proofpoint_tap.message_delivered-logs number_of_replicas: 0 + composed_of: + - "logs-proofpoint_tap.message_delivered@package" + - "logs-proofpoint_tap.message_delivered@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -8258,8 +8124,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-pulse_connect_secure.log@custom index_patterns: - logs-pulse_connect_secure.log-* priority: 501 @@ -8304,8 +8168,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-sentinel_one.activity@custom index_patterns: - logs-sentinel_one.activity-* priority: 501 @@ -8350,8 +8212,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-sentinel_one.agent@custom index_patterns: - logs-sentinel_one.agent-* priority: 501 @@ -8396,8 +8256,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-sentinel_one.alert@custom index_patterns: - logs-sentinel_one.alert-* priority: 501 @@ -8442,8 +8300,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-sentinel_one.group@custom index_patterns: - logs-sentinel_one.group-* priority: 501 @@ -8488,8 +8344,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-sentinel_one.threat@custom index_patterns: - logs-sentinel_one.threat-* priority: 501 @@ -8524,27 +8378,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-snort_x_log: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-snort.log@package - - logs-snort.log@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-snort.log@custom index_patterns: - - logs-snort.log-* - priority: 501 + - "logs-snort.log-*" template: settings: index: lifecycle: name: so-logs-snort.log-logs number_of_replicas: 0 + composed_of: + - "logs-snort.log@package" + - "logs-snort.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -8580,8 +8432,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-snyk.audit@custom index_patterns: - logs-snyk.audit-* priority: 501 @@ -8626,8 +8476,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-snyk.vulnerabilities@custom index_patterns: - logs-snyk.vulnerabilities-* priority: 501 @@ -8661,114 +8509,6 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-logs-soc: - close: 30 - delete: 365 - index_sorting: false - index_template: - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - container-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - common-settings - - common-dynamic-mappings - data_stream: {} - ignore_missing_component_templates: [] - index_patterns: - - logs-soc-so* - priority: 500 - template: - mappings: - date_detection: false - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - settings: - index: - lifecycle: - name: so-logs-soc-logs - mapping: - total_fields: - limit: 5000 - number_of_replicas: 0 - number_of_shards: 1 - refresh_interval: 30s - sort: - field: '@timestamp' - order: desc - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - warm: 7 so-logs-sonicwall_firewall_x_log: index_sorting: false index_template: @@ -8780,8 +8520,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-sonicwall_firewall.log@custom index_patterns: - logs-sonicwall_firewall.log-* priority: 501 @@ -8826,8 +8564,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-sophos_central.alert@custom index_patterns: - logs-sophos_central.alert-* priority: 501 @@ -8872,8 +8608,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-sophos_central.event@custom index_patterns: - logs-sophos_central.event-* priority: 501 @@ -8918,8 +8652,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-sophos.utm@custom index_patterns: - logs-sophos.utm-* priority: 501 @@ -8964,8 +8696,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-sophos.xg@custom index_patterns: - logs-sophos.xg-* priority: 501 @@ -9010,8 +8740,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-symantec_endpoint.log@custom index_patterns: - logs-symantec_endpoint.log-* priority: 501 @@ -9057,8 +8785,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-system.application@custom index_patterns: - logs-system.application* priority: 501 @@ -9104,8 +8830,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-system.auth@custom index_patterns: - logs-system.auth* priority: 501 @@ -9151,8 +8875,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-system.security@custom index_patterns: - logs-system.security* priority: 501 @@ -9198,8 +8920,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-system.syslog@custom index_patterns: - logs-system.syslog* priority: 501 @@ -9245,8 +8965,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-system.system@custom index_patterns: - logs-system.system* priority: 501 @@ -9291,8 +9009,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-tenable_sc.asset@custom index_patterns: - logs-tenable_sc.asset-* priority: 501 @@ -9337,8 +9053,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-tenable_sc.plugin@custom index_patterns: - logs-tenable_sc.plugin-* priority: 501 @@ -9383,8 +9097,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-tenable_sc.vulnerability@custom index_patterns: - logs-tenable_sc.vulnerability-* priority: 501 @@ -9429,8 +9141,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-ti_abusech.malware@custom index_patterns: - logs-ti_abusech.malware-* priority: 501 @@ -9475,8 +9185,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-ti_abusech.malwarebazaar@custom index_patterns: - logs-ti_abusech.malwarebazaar-* priority: 501 @@ -9521,8 +9229,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-ti_abusech.threatfox@custom index_patterns: - logs-ti_abusech.threatfox-* priority: 501 @@ -9567,8 +9273,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-ti_abusech.url@custom index_patterns: - logs-ti_abusech.url-* priority: 501 @@ -9603,27 +9307,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-ti_anomali_x_threatstream: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-ti_anomali.threatstream@package - - logs-ti_anomali.threatstream@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-ti_anomali.threatstream@custom index_patterns: - - logs-ti_anomali.threatstream-* - priority: 501 + - "logs-ti_anomali.threatstream-*" template: settings: index: lifecycle: name: so-logs-ti_anomali.threatstream-logs number_of_replicas: 0 + composed_of: + - "logs-ti_anomali.threatstream@package" + - "logs-ti_anomali.threatstream@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -9649,27 +9351,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-ti_cybersixgill_x_threat: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-ti_cybersixgill.threat@package - - logs-ti_cybersixgill.threat@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-ti_cybersixgill.threat@custom index_patterns: - - logs-ti_cybersixgill.threat-* - priority: 501 + - "logs-ti_cybersixgill.threat-*" template: settings: index: lifecycle: name: so-logs-ti_cybersixgill.threat-logs number_of_replicas: 0 + composed_of: + - "logs-ti_cybersixgill.threat@package" + - "logs-ti_cybersixgill.threat@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -9705,8 +9405,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-ti_misp.threat@custom index_patterns: - logs-ti_misp.threat-* priority: 501 @@ -9751,8 +9449,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-ti_misp.threat_attributes@custom index_patterns: - logs-ti_misp.threat_attributes-* priority: 501 @@ -9797,8 +9493,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-ti_otx.pulses_subscribed@custom index_patterns: - logs-ti_otx.pulses_subscribed-* priority: 501 @@ -9843,8 +9537,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-ti_otx.threat@custom index_patterns: - logs-ti_otx.threat-* priority: 501 @@ -9889,8 +9581,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-ti_recordedfuture.latest_ioc-template@custom index_patterns: - logs-ti_recordedfuture.latest_ioc-template-* priority: 501 @@ -9935,8 +9625,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-ti_recordedfuture.threat@custom index_patterns: - logs-ti_recordedfuture.threat-* priority: 501 @@ -9971,27 +9659,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-ti_threatq_x_threat: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-ti_threatq.threat@package - - logs-ti_threatq.threat@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-ti_threatq.threat@custom index_patterns: - - logs-ti_threatq.threat-* - priority: 501 + - "logs-ti_threatq.threat-*" template: settings: index: lifecycle: name: so-logs-ti_threatq.threat-logs number_of_replicas: 0 + composed_of: + - "logs-ti_threatq.threat@package" + - "logs-ti_threatq.threat@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -10017,27 +9703,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-vsphere_x_log: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-vsphere.log@package - - logs-vsphere.log@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-vsphere.log@custom index_patterns: - - logs-vsphere.log-* - priority: 501 + - "logs-vsphere.log-*" template: settings: index: lifecycle: name: so-logs-vsphere.log-logs number_of_replicas: 0 + composed_of: + - "logs-vsphere.log@package" + - "logs-vsphere.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -10073,8 +9757,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-windows.forwarded@custom index_patterns: - logs-windows.forwarded* priority: 501 @@ -10119,8 +9801,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-windows.powershell@custom index_patterns: - logs-windows.powershell-* priority: 501 @@ -10165,8 +9845,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-windows.powershell_operational@custom index_patterns: - logs-windows.powershell_operational-* priority: 501 @@ -10211,8 +9889,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-windows.sysmon_operational@custom index_patterns: - logs-windows.sysmon_operational-* priority: 501 @@ -10247,28 +9923,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-winlog_x_winlog: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-winlog.winlog@package - - logs-winlog.winlog@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-winlog.winlog@package - - logs-winlog.winlog@custom index_patterns: - - logs-winlog.winlog-* - priority: 501 + - "logs-winlog.winlog-*" template: settings: index: lifecycle: name: so-logs-winlog.winlog-logs number_of_replicas: 0 + composed_of: + - "logs-winlog.winlog@package" + - "logs-winlog.winlog@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -10304,8 +9977,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-zscaler_zia.alerts@custom index_patterns: - logs-zscaler_zia.alerts-* priority: 501 @@ -10350,8 +10021,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-zscaler_zia.dns@custom index_patterns: - logs-zscaler_zia.dns-* priority: 501 @@ -10396,8 +10065,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-zscaler_zia.firewall@custom index_patterns: - logs-zscaler_zia.firewall-* priority: 501 @@ -10442,8 +10109,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-zscaler_zia.tunnel@custom index_patterns: - logs-zscaler_zia.tunnel-* priority: 501 @@ -10488,8 +10153,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-zscaler_zia.web@custom index_patterns: - logs-zscaler_zia.web-* priority: 501 @@ -10534,8 +10197,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-zscaler_zpa.app_connector_status@custom index_patterns: - logs-zscaler_zpa.app_connector_status-* priority: 501 @@ -10580,8 +10241,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-zscaler_zpa.audit@custom index_patterns: - logs-zscaler_zpa.audit-* priority: 501 @@ -10626,8 +10285,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-zscaler_zpa.browser_access@custom index_patterns: - logs-zscaler_zpa.browser_access-* priority: 501 @@ -10672,8 +10329,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-zscaler_zpa.user_activity@custom index_patterns: - logs-zscaler_zpa.user_activity-* priority: 501 @@ -10718,8 +10373,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-zscaler_zpa.user_status@custom index_patterns: - logs-zscaler_zpa.user_status-* priority: 501 @@ -10753,6 +10406,317 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-metrics-endpoint_x_metadata: + index_sorting: False + index_template: + index_patterns: + - "metrics-endpoint.metadata-*" + template: + settings: + index: + lifecycle: + name: so-metrics-endpoint.metadata-logs + number_of_replicas: 0 + composed_of: + - "metrics-endpoint.metadata@package" + - "metrics-endpoint.metadata@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-endpoint_x_metrics: + index_sorting: False + index_template: + index_patterns: + - "metrics-endpoint.metrics-*" + template: + settings: + index: + lifecycle: + name: so-metrics-endpoint.metrics-logs + number_of_replicas: 0 + composed_of: + - "metrics-endpoint.metrics@package" + - "metrics-endpoint.metrics@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-endpoint_x_policy: + index_sorting: False + index_template: + index_patterns: + - "metrics-endpoint.policy-*" + template: + settings: + index: + lifecycle: + name: so-metrics-endpoint.policy-logs + number_of_replicas: 0 + composed_of: + - "metrics-endpoint.policy@package" + - "metrics-endpoint.policy@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-vsphere_x_datastore: + index_sorting: False + index_template: + index_patterns: + - "metrics-vsphere.datastore-*" + template: + settings: + index: + lifecycle: + name: so-metrics-vsphere.datastore-logs + number_of_replicas: 0 + composed_of: + - "metrics-vsphere.datastore@package" + - "metrics-vsphere.datastore@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-vsphere_x_host: + index_sorting: False + index_template: + index_patterns: + - "metrics-vsphere.host-*" + template: + settings: + index: + lifecycle: + name: so-metrics-vsphere.host-logs + number_of_replicas: 0 + composed_of: + - "metrics-vsphere.host@package" + - "metrics-vsphere.host@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-vsphere_x_virtualmachine: + index_sorting: False + index_template: + index_patterns: + - "metrics-vsphere.virtualmachine-*" + template: + settings: + index: + lifecycle: + name: so-metrics-vsphere.virtualmachine-logs + number_of_replicas: 0 + composed_of: + - "metrics-vsphere.virtualmachine@package" + - "metrics-vsphere.virtualmachine@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-kismet: + index_sorting: false + index_template: + composed_of: + - kismet-mappings + - source-mappings + - client-mappings + - device-mappings + - network-mappings + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-kismet-so* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-kismet-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logstash: index_sorting: false index_template: @@ -10814,7 +10778,6 @@ elasticsearch: - vulnerability-mappings - common-settings - common-dynamic-mappings - ignore_missing_component_templates: [] index_patterns: - logs-logstash-default* priority: 500 @@ -10864,374 +10827,6 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-metrics-endpoint_x_metadata: - index_sorting: false - index_template: - composed_of: - - metrics-endpoint.metadata@package - - metrics-endpoint.metadata@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - metrics-endpoint.metadata@custom - index_patterns: - - metrics-endpoint.metadata-* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-metrics-endpoint.metadata-logs - number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-endpoint_x_metrics: - index_sorting: false - index_template: - composed_of: - - metrics-endpoint.metrics@package - - metrics-endpoint.metrics@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - metrics-endpoint.metrics@custom - index_patterns: - - metrics-endpoint.metrics-* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-metrics-endpoint.metrics-logs - number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-endpoint_x_policy: - index_sorting: false - index_template: - composed_of: - - metrics-endpoint.policy@package - - metrics-endpoint.policy@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - metrics-endpoint.policy@custom - index_patterns: - - metrics-endpoint.policy-* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-metrics-endpoint.policy-logs - number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-fleet_server_x_agent_status: - index_sorting: false - index_template: - composed_of: - - metrics@tsdb-settings - - metrics-fleet_server.agent_status@package - - metrics-fleet_server.agent_status@custom - - ecs@mappings - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - metrics-fleet_server.agent_status@custom - index_patterns: - - metrics-fleet_server.agent_status-* - priority: 501 - template: - settings: - index: - mode: time_series - number_of_replicas: 0 - so-metrics-fleet_server_x_agent_versions: - index_sorting: false - index_template: - composed_of: - - metrics@tsdb-settings - - metrics-fleet_server.agent_versions@package - - metrics-fleet_server.agent_versions@custom - - ecs@mappings - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - metrics-fleet_server.agent_versions@custom - index_patterns: - - metrics-fleet_server.agent_versions-* - priority: 501 - template: - settings: - index: - mode: time_series - number_of_replicas: 0 - so-metrics-nginx_x_stubstatus: - index_sorting: false - index_template: - composed_of: - - metrics-nginx.stubstatus@package - - metrics-nginx.stubstatus@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - metrics-nginx.stubstatus@custom - index_patterns: - - metrics-nginx.stubstatus-* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-metrics-nginx.stubstatus-logs - number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-vsphere_x_datastore: - index_sorting: false - index_template: - composed_of: - - metrics-vsphere.datastore@package - - metrics-vsphere.datastore@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - metrics-vsphere.datastore@custom - index_patterns: - - metrics-vsphere.datastore-* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-metrics-vsphere.datastore-logs - number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-vsphere_x_host: - index_sorting: false - index_template: - composed_of: - - metrics-vsphere.host@package - - metrics-vsphere.host@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - metrics-vsphere.host@custom - index_patterns: - - metrics-vsphere.host-* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-metrics-vsphere.host-logs - number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-vsphere_x_virtualmachine: - index_sorting: false - index_template: - composed_of: - - metrics-vsphere.virtualmachine@package - - metrics-vsphere.virtualmachine@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - metrics-vsphere.virtualmachine@custom - index_patterns: - - metrics-vsphere.virtualmachine-* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-metrics-vsphere.virtualmachine-logs - number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d so-redis: index_sorting: false index_template: @@ -11293,7 +10888,6 @@ elasticsearch: - vulnerability-mappings - common-settings - common-dynamic-mappings - ignore_missing_component_templates: [] index_patterns: - logs-redis-default* priority: 500 @@ -11406,7 +11000,6 @@ elasticsearch: - common-settings - common-dynamic-mappings data_stream: {} - ignore_missing_component_templates: [] index_patterns: - logs-strelka-so* priority: 500 @@ -11518,7 +11111,6 @@ elasticsearch: - common-settings - common-dynamic-mappings data_stream: {} - ignore_missing_component_templates: [] index_patterns: - logs-suricata-so* priority: 500 @@ -11630,7 +11222,6 @@ elasticsearch: - common-settings - common-dynamic-mappings data_stream: {} - ignore_missing_component_templates: [] index_patterns: - logs-suricata.alerts-* priority: 500 @@ -11743,7 +11334,6 @@ elasticsearch: - common-settings - common-dynamic-mappings data_stream: {} - ignore_missing_component_templates: [] index_patterns: - logs-syslog-so* priority: 500 @@ -11857,7 +11447,6 @@ elasticsearch: - common-settings - common-dynamic-mappings data_stream: {} - ignore_missing_component_templates: [] index_patterns: - logs-zeek-so* priority: 500 @@ -11907,87 +11496,6 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - pipelines: - custom001: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom001 - - pipeline: - name: common - custom002: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom002 - - pipeline: - name: common - custom003: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom003 - - pipeline: - name: common - custom004: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom004 - - pipeline: - name: common - custom005: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom005 - - pipeline: - name: common - custom006: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom006 - - pipeline: - name: common - custom007: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom007 - - pipeline: - name: common - custom008: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom008 - - pipeline: - name: common - custom009: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom009 - - pipeline: - name: common - custom010: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom010 - - pipeline: - name: common retention: retention_pct: 50 so_roles: diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 085aab7f0..f56ed313e 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -530,58 +530,6 @@ elasticsearch: so-strelka: *indexSettings so-syslog: *indexSettings so-zeek: *indexSettings - so-metrics-fleet_server_x_agent_status: &fleetMetricsSettings - index_sorting: - description: Sorts the index by event time, at the cost of additional processing resource consumption. - advanced: True - readonly: True - helpLink: elasticsearch.html - index_template: - ignore_missing_component_templates: - description: Ignore component templates if they aren't in Elasticsearch. - advanced: True - readonly: True - helpLink: elasticsearch.html - index_patterns: - description: Patterns for matching multiple indices or tables. - advanced: True - readonly: True - helpLink: elasticsearch.html - template: - settings: - index: - mode: - description: Type of mode used for this index. Time series indices can be used for metrics to reduce necessary storage. - advanced: True - readonly: True - helpLink: elasticsearch.html - number_of_replicas: - description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs. - advanced: True - readonly: True - helpLink: elasticsearch.html - composed_of: - description: The index template is composed of these component templates. - advanced: True - readonly: True - helpLink: elasticsearch.html - priority: - description: The priority of the index template. - advanced: True - readonly: True - helpLink: elasticsearch.html - data_stream: - hidden: - description: Hide the data stream. - advanced: True - readonly: True - helpLink: elasticsearch.html - allow_custom_routing: - description: Allow custom routing for the data stream. - advanced: True - readonly: True - helpLink: elasticsearch.html - so-metrics-fleet_server_x_agent_versions: *fleetMetricsSettings so_roles: so-manager: &soroleSettings config: diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load index 12ef4dbf6..080348522 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -134,7 +134,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then TEMPLATE=${i::-14} COMPONENT_PATTERN=${TEMPLATE:3} MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery") - if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ logs-http_endpoint\.generic|logs-winlog\.winlog ]]; then + if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" ]]; then load_failures=$((load_failures+1)) echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures" else @@ -153,7 +153,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then cd - >/dev/null if [[ $load_failures -eq 0 ]]; then - echo "All templates loaded successfully" + echo "All template loaded successfully" touch $STATE_FILE_SUCCESS else echo "Encountered $load_failures templates that were unable to load, likely due to missing dependencies that will be available later; will retry on next highstate" diff --git a/salt/kibana/files/config_saved_objects.ndjson b/salt/kibana/files/config_saved_objects.ndjson index 50e2ba45c..bc503debb 100644 --- a/salt/kibana/files/config_saved_objects.ndjson +++ b/salt/kibana/files/config_saved_objects.ndjson @@ -1 +1 @@ -{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.14.2","id": "8.14.2","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} +{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.10.4","id": "8.10.4","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} diff --git a/salt/kibana/tools/sbin_jinja/so-kibana-config-load b/salt/kibana/tools/sbin_jinja/so-kibana-config-load index fc0896009..5b4b52560 100644 --- a/salt/kibana/tools/sbin_jinja/so-kibana-config-load +++ b/salt/kibana/tools/sbin_jinja/so-kibana-config-load @@ -63,7 +63,7 @@ update() { IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' for i in "${LINES[@]}"; do - RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.14.2" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") + RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.10.4" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi done From a23789287e86033a9e35726f8a1c4b9f6d69102a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 15 Jul 2024 11:29:47 -0400 Subject: [PATCH 34/38] force var to be list of string --- salt/suricata/soc_suricata.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index 1ecabacd8..f865468e1 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -154,12 +154,14 @@ suricata: description: Assign a list of hosts, or networks, using CIDR notation, to this Suricata variable. The variable can then be re-used within Suricata rules. This allows for a single adjustment to the variable that will then affect all rules referencing the variable. regex: ^(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\/([0-9]|[1-2][0-9]|3[0-2]))?$|^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?))|:))|(([0-9A-Fa-f]{1,4}:){5}((:[0-9A-Fa-f]{1,4}){1,2}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){4}((:[0-9A-Fa-f]{1,4}){1,3}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){3}((:[0-9A-Fa-f]{1,4}){1,4}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){2}((:[0-9A-Fa-f]{1,4}){1,5}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){1}((:[0-9A-Fa-f]{1,4}){1,6}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(:((:[0-9A-Fa-f]{1,4}){1,7}|:)))(\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8]))?$ regexFailureMessage: You must enter a valid IP address or CIDR. - helpLink: suricata.html + forcedType: "[]string" duplicates: True + helpLink: suricata.html EXTERNAL_NET: &suriaddressgroup description: Assign a list of hosts, or networks, or other customization, to this Suricata variable. The variable can then be re-used within Suricata rules. This allows for a single adjustment to the variable that will then affect all rules referencing the variable. - helpLink: suricata.html + forcedType: "[]string" duplicates: True + helpLink: suricata.html HTTP_SERVERS: *suriaddressgroup SMTP_SERVERS: *suriaddressgroup SQL_SERVERS: *suriaddressgroup @@ -176,8 +178,9 @@ suricata: port-groups: HTTP_PORTS: &suriportgroup description: Assign a list of network port numbers to this Suricata variable. The variable can then be re-used within Suricata rules. This allows for a single adjustment to the variable that will then affect all rules referencing the variable. - helpLink: suricata.html + forcedType: "[]string" duplicates: True + helpLink: suricata.html SHELLCODE_PORTS: *suriportgroup ORACLE_PORTS: *suriportgroup SSH_PORTS: *suriportgroup From 4e81860a13486d272b9395301822f0b2b69da4c4 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 15 Jul 2024 11:33:52 -0400 Subject: [PATCH 35/38] Revert "Change pipeline version for agent" --- .../component/elastic-agent/logs-elastic_agent@package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json index efd85bb4b..7bcc34de9 100644 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json +++ b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json @@ -6,7 +6,7 @@ "name": "logs" }, "codec": "best_compression", - "default_pipeline": "logs-elastic_agent-1.20.0", + "default_pipeline": "logs-elastic_agent-1.13.1", "mapping": { "total_fields": { "limit": "10000" From 3991c7b5fea285f2277a4a03a4165b720eb294ac Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Mon, 15 Jul 2024 15:52:00 -0400 Subject: [PATCH 36/38] FEATURE: Add new action to SOC Actions list to allow users to more easily add their own actions #13346 --- salt/soc/defaults.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index ce183b862..6ddf534c7 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -96,6 +96,12 @@ soc: links: - '/#/alerts?q=rule.uuid: {:so_detection.publicId|escape} | groupby rule.name event.module* event.severity_label' target: '' + - name: actionAdd + description: actionAddHelp + icon: fa-plus + links: + - '/#/config?s=soc.config.actions' + target: '' eventFields: default: - soc_timestamp From d0565baaa32149985178f6e41b75aa1587551cc8 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Mon, 15 Jul 2024 14:43:47 -0600 Subject: [PATCH 37/38] New Config Values for Detections Bulk Indexer `maxScrollSize` defines the "page size" of each scroll request. `bulkIndexerWorkerCount` defines how many worker threads a bulk indexer should use. 0 or fewer indicates that 1 thread per CPU core should be used. --- salt/soc/defaults.yaml | 12 +++++++----- salt/soc/soc_soc.yaml | 4 ++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 6ddf534c7..6d3667d0b 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1366,6 +1366,8 @@ soc: maxLogLength: 1024 asyncThreshold: 10 lookupTunnelParent: true + maxScrollSize: 10000 + bulkIndexerWorkerCount: -1 influxdb: hostUrl: token: @@ -2292,15 +2294,15 @@ soc: alert http $EXTERNAL_NET any -> $HOME_NET any (msg:"Example Rule Title - 'example' String Detected"; content:"example"; sid:[publicId]; rev:1;) strelka: | - /* + /* This is a YARA rule template. Replace all template values with your own values. The YARA rule name is the unique identifier for the rule. Docs: https://yara.readthedocs.io/en/stable/writingrules.html#writing-yara-rules - */ + */ rule Example // This identifier _must_ be unique { - meta: + meta: description = "Generic YARA Rule" author = "@SecurityOnion" date = "YYYY-MM-DD" @@ -2323,7 +2325,7 @@ soc: id: [publicId] status: 'experimental' description: | - This should be a detailed description of what this Detection focuses on: what we are trying to find and why we are trying to find it. + This should be a detailed description of what this Detection focuses on: what we are trying to find and why we are trying to find it. For example, from rule 97a80ec7-0e2f-4d05-9ef4-65760e634f6b: "Detects a whoami.exe executed with the /priv command line flag instructing the tool to show all current user privileges. This is often used after a privilege escalation attempt." references: - 'https://local.invalid' @@ -2332,7 +2334,7 @@ soc: tags: - detection.threat_hunting - attack.technique_id - logsource: + logsource: category: process_creation product: windows detection: diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 4975b75f0..35402f760 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -174,6 +174,10 @@ soc: lookupTunnelParent: description: When true, if a pivoted event appears to be encapsulated, such as in a VXLAN packet, then SOC will pivot to the VXLAN packet stream. When false, SOC will attempt to pivot to the encapsulated packet stream itself, but at the risk that it may be unable to locate it in the stored PCAP data. global: True + maxScrollSize: + description: The maximum number of documents to request in a single Elasticsearch scroll request. + bulkIndexWorkerCount: + description: The number of worker threads to use when bulk indexing data into Elasticsearch. A value below 1 will default to the number of CPUs available. sostatus: refreshIntervalMs: description: Duration (in milliseconds) between refreshes of the grid status. Shortening this duration may not have expected results, as the backend systems feeding this sostatus data will continue their updates as scheduled. From b814f32e0aab3742ac3f16776330763d18a242e9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 16 Jul 2024 12:39:30 -0400 Subject: [PATCH 38/38] fix custom indices --- salt/elasticsearch/config.map.jinja | 5 +++++ salt/elasticsearch/template.map.jinja | 17 +++++++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 8127df848..f6062e1f2 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -1,3 +1,8 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at + https://securityonion.net/license; you may not use this file except in compliance with the + Elastic License 2.0. #} + {% from 'vars/globals.map.jinja' import GLOBALS %} {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS with context %} diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index b59c291a4..06041b537 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -1,3 +1,8 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at + https://securityonion.net/license; you may not use this file except in compliance with the + Elastic License 2.0. #} + {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %} {% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %} @@ -17,10 +22,14 @@ {% set ES_INDEX_SETTINGS = {} %} {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %} {% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %} -{# if policy isn't defined in the original index settings, then dont merge policy from the global_overrides #} -{# this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #} -{% if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %} -{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %} +{# prevent this action from being performed on custom defined indices. #} +{# the custom defined index is not present in either of the dictionaries and fails to reder. #} +{% if index in ES_INDEX_SETTINGS_ORIG and index in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES %} +{# dont merge policy from the global_overrides if policy isn't defined in the original index settingss #} +{# this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #} +{% if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %} +{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %} +{% endif %} {% endif %} {% if settings.index_template is defined %}