From ae45d40eca779ebb06403b3ce5ffabc1572f32e9 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 1 Nov 2023 13:34:30 +0000 Subject: [PATCH 01/11] Add Sublime Platform ingest pipeline --- salt/elasticsearch/files/ingest/sublime | 34 +++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 salt/elasticsearch/files/ingest/sublime diff --git a/salt/elasticsearch/files/ingest/sublime b/salt/elasticsearch/files/ingest/sublime new file mode 100644 index 000000000..c26f93c01 --- /dev/null +++ b/salt/elasticsearch/files/ingest/sublime @@ -0,0 +1,34 @@ +{ + "description" : " Email alerts from Sublime", + "processors" : [ + { "set": { "field": "event.module", "value": "sublime" } }, + { "set": { "field": "event.dataset", "value": "alert" } }, + { "set": { "field": "event.severity", "value": 3, "override": true } }, + { "set": { "field": "rule.name", "value": "Sublime Platform: {{ flagged_rules.0.name }}", "override": true } }, + { "set": { "field": "sublime.message_group_id", "value": "{{ _id }}", "override": true } }, + { "set": { "field": "email.address", "value": "{{ messages.0.recipients.0.email }}", "override": true } }, + { "set": { "field": "email.forwarded_recipents", "value": "{{ messages.0.forwarded_receipients }}", "override": true } }, + { "set": { "field": "email.sender.address", "value": "{{ messages.0.sender.email }}", "override": true } }, + { "set": { "field": "email.subject", "value": "{{ messages.0.subject }}", "override": true } }, + { "set": { "field": "email.forwarded_at", "value": "{{ messages.0.forwarded_at }}", "override": true } }, + { "set": { "field": "email.created_at", "value": "{{ messages.0.created_at }}", "override": true } }, + { "set": { "field": "email.read_at", "value": "{{ messages.0.read_at }}", "override": true } }, + { "set": { "field": "email.replied_at", "value": "{{ messages.0.replied_at }}", "override": true } }, + { + "grok": { + "field": "sublime.request_url", + "patterns": ["^https://api.%{DATA:sublime_host}/v0%{GREEDYDATA}$"], + "ignore_failure": true + } + }, + + { "rename": { "field": "sublime_host", "target_field": "sublime.url", "ignore_missing": true } }, + { "rename": { "field": "data", "target_field": "sublime", "ignore_missing": true } }, + { "rename": { "field": "flagged_rules", "target_field": "sublime.flagged_rules", "ignore_missing": true } }, + { "rename": { "field": "organization_id", "target_field": "sublime.organization_id", "ignore_missing": true } }, + { "rename": { "field": "review_status", "target_field": "sublime.review_status", "ignore_missing": true } }, + { "rename": { "field": "state", "target_field": "sublime.state", "ignore_missing": true } }, + { "rename": { "field": "user_reports", "target_field": "sublime.user_reports", "ignore_missing": true } }, + { "pipeline": { "name": "common" } } + ] +} From 4dc64400c58b9de09f6ac811180a55208dcea7a6 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 1 Nov 2023 13:36:32 +0000 Subject: [PATCH 02/11] Support document_id --- .../so/9805_output_elastic_agent.conf.jinja | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja b/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja index 0a148155c..f7671e2b7 100644 --- a/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja +++ b/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja @@ -1,13 +1,16 @@ output { if "elastic-agent" in [tags] { - if [metadata][pipeline] { + if [metadata][pipeline] { + if [metadata][_id] { elasticsearch { hosts => "{{ GLOBALS.manager }}" ecs_compatibility => v8 data_stream => true user => "{{ ES_USER }}" password => "{{ ES_PASS }}" + document_id => "%{[metadata][_id]}" pipeline => "%{[metadata][pipeline]}" + silence_errors_in_log => ["version_conflict_engine_exception"] ssl => true ssl_certificate_verification => false } @@ -19,10 +22,22 @@ output { data_stream => true user => "{{ ES_USER }}" password => "{{ ES_PASS }}" + pipeline => "%{[metadata][pipeline]}" ssl => true - ssl_certificate_verification => false + ssl_certificate_verification => false } - } + } + } + else { + elasticsearch { + hosts => "{{ GLOBALS.manager }}" + ecs_compatibility => v8 + data_stream => true + user => "{{ ES_USER }}" + password => "{{ ES_PASS }}" + ssl => true + ssl_certificate_verification => false + } + } } } - From 51247be6b97583e15dad1af1c93ff031ef0a51a9 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 1 Nov 2023 13:37:52 +0000 Subject: [PATCH 03/11] Sublime Platform integration defaults --- salt/elasticfleet/defaults.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index a17957e7c..ba779f5a0 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -90,3 +90,10 @@ elasticfleet: - zscaler_zia - zscaler_zpa - 1password + optional_integrations: + sublime_platform: + enabled_nodes: [] + api_key: + base_url: https://api.platform.sublimesecurity.com + poll_interval: 5m + limit: 100 From 23ee9c2bb02842b7cd745e7b667bc02ad4ea6328 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 1 Nov 2023 13:41:40 +0000 Subject: [PATCH 04/11] Sublime Platform integration --- .../sublime_platform.json | 44 +++++++++++++++++++ salt/elasticfleet/soc_elasticfleet.yaml | 32 ++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 salt/elasticfleet/files/integrations-optional/sublime_platform.json diff --git a/salt/elasticfleet/files/integrations-optional/sublime_platform.json b/salt/elasticfleet/files/integrations-optional/sublime_platform.json new file mode 100644 index 000000000..8feedc879 --- /dev/null +++ b/salt/elasticfleet/files/integrations-optional/sublime_platform.json @@ -0,0 +1,44 @@ +{%- from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED -%} +{%- from 'sensoroni/map.jinja' import SENSORONIMERGED -%} +{%- from 'vars/globals.map.jinja' import GLOBALS -%} +{%- raw -%} +{ + "package": { + "name": "httpjson", + "version": "" + }, + "name": "sublime-platform", + "namespace": "default", + "description": "", + "policy_id": "FleetServer_{%- endraw -%}{{ NAME }}{%- raw -%}", + "vars": {}, + "inputs": { + "generic-httpjson": { + "enabled": true, + "streams": { + "httpjson.generic": { + "enabled": true, + "vars": { + "request_method": "GET", + "processors": "- drop_event:\n when:\n not:\n contains: \n message: \"flagged_rules\"\n- decode_json_fields:\n fields: [\"message\"]\n document_id: id\n target: \"\"", + "enable_request_tracer": false, + "oauth_scopes": [], + "request_transforms": "- set:\n target: header.Authorization\n value: 'Bearer {% endraw -%}{{ ELASTICFLEETMERGED.optional_integrations.sublime_platform.api_key }}{%- raw -%}'\n- set:\n target: header.accept\n value: application/json\n- set:\n target: url.params.last_message_created_at[gte]\n value: '[[formatDate (now (parseDuration \"-{%- endraw -%}{{ ELASTICFLEETMERGED.optional_integrations.sublime_platform.poll_interval }}{%- raw -%}\")) \"2006-01-02T15:04:05Z\"]]'\n- set:\n target: url.params.reviewed\n value: false\n- set:\n target: url.params.flagged\n value: true\n- set:\n target: url.params.limit\n value: {% endraw %}{{ ELASTICFLEETMERGED.optional_integrations.sublime_platform.limit }}{%- raw -%}", + "response_transforms": "", + "request_redirect_headers_ban_list": [], + "request_encode_as": "application/x-www-form-urlencoded", + "request_url": "{%- endraw -%}{{ ELASTICFLEETMERGED.optional_integrations.sublime_platform.base_url }}{%- raw -%}/v0/message-groups", + "response_split": "target: body.message_groups\ntype: array\nkeep_parent: false\ntransforms:\n - set:\n target: body.sublime.request_url\n value : '[[ .last_response.url.value ]]'", + "tags": [ + "forwarded" + ], + "pipeline": "sublime", + "data_stream.dataset": "sublime", + "request_interval": "1m" + } + } + } + } + } +} +{%- endraw -%} diff --git a/salt/elasticfleet/soc_elasticfleet.yaml b/salt/elasticfleet/soc_elasticfleet.yaml index af660358a..8cb975086 100644 --- a/salt/elasticfleet/soc_elasticfleet.yaml +++ b/salt/elasticfleet/soc_elasticfleet.yaml @@ -40,3 +40,35 @@ elasticfleet: helpLink: elastic-fleet.html sensitive: True advanced: True + optional_integrations: + sublime_platform: + enabled_nodes: + description: Determines if the Sublime Platform integration is enabled. + global: True + helpLink: elastic-fleet.html + advanced: True + forcedType: "[]string" + api_key: + description: API key for Sublime Platform. + global: False + helpLink: elastic-fleet.html + advanced: True + forcedType: string + base_url: + description: Base URL for Sublime Platform. + global: False + helpLink: elastic-fleet.html + advanced: True + forcedType: string + poll_interval: + description: Poll interval for alerts from Sublime Platform. + global: False + helpLink: elastic-fleet.html + advanced: True + forcedType: string + limit: + description: The maximum number of message groups to return from Sublime Platform. + global: False + helpLink: elastic-fleet.html + advanced: True + forcedType: int From 9701d0ac206382124b5e8ffe854267b30f48aa18 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 1 Nov 2023 13:47:20 +0000 Subject: [PATCH 05/11] Optional integration Fleet configuration --- salt/elasticfleet/config.sls | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/salt/elasticfleet/config.sls b/salt/elasticfleet/config.sls index d2e357c91..29eb6e972 100644 --- a/salt/elasticfleet/config.sls +++ b/salt/elasticfleet/config.sls @@ -6,6 +6,7 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} {% if sls.split('.')[0] in allowed_states %} +{% set node_data = salt['pillar.get']('node_data') %} # Add EA Group elasticfleetgroup: @@ -92,6 +93,34 @@ eaintegration: - user: 947 - group: 939 +{% for minion in node_data %} +{% set role = node_data[minion]["role"] %} +{% if role in [ "fleet","heavynode", "manager","managersearch","standalone" ] %} +{% set optional_integrations = salt['pillar.get']('elasticfleet:optional_integrations', {}) %} +{% set integration_keys = salt['pillar.get']('elasticfleet:optional_integrations', {}).keys() %} +fleet_server_integrations_{{ minion }}: + file.directory: + - name: /opt/so/conf/elastic-fleet/integrations/FleetServer_{{ minion }} + - user: 947 + - group: 939 + - makedirs: True +{% for integration in integration_keys %} +{% set enabled_nodes = optional_integrations[integration]["enabled_nodes"] %} +{% if minion in enabled_nodes %} +optional_integrations_dynamic_{{ minion }}: + file.managed: + - name: /opt/so/conf/elastic-fleet/integrations/FleetServer_{{ minion }}/{{ integration }}.json + - source: salt://elasticfleet/files/integrations-optional/{{ integration }}.json + - user: 947 + - group: 939 + - template: jinja + - defaults: + NAME: {{ minion }} +{% endif %} +{% endfor %} +{% endif %} +{% endfor %} + ea-integrations-load: file.absent: - name: /opt/so/state/eaintegrations.txt @@ -99,6 +128,9 @@ ea-integrations-load: - file: eaintegration - file: eadynamicintegration - file: eapackageupgrade + {% for minion in node_data %} + - file: optional_integrations_dynamic_{{ minion }} + {% endfor %} {% endif %} {% else %} From 44e45843bfc5c9af30f54ea01306e2ee5a2408d2 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 1 Nov 2023 13:52:38 +0000 Subject: [PATCH 06/11] Change optional integration Fleet configuration --- salt/elasticfleet/config.sls | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/salt/elasticfleet/config.sls b/salt/elasticfleet/config.sls index 29eb6e972..78fb1d412 100644 --- a/salt/elasticfleet/config.sls +++ b/salt/elasticfleet/config.sls @@ -93,6 +93,13 @@ eaintegration: - user: 947 - group: 939 +eaoptionalintegrationsdir: + file.directory: + - name: /opt/so/conf/elastic-fleet/integrations-optional + - user: 947 + - group: 939 + - makedirs: True + {% for minion in node_data %} {% set role = node_data[minion]["role"] %} {% if role in [ "fleet","heavynode", "manager","managersearch","standalone" ] %} @@ -100,16 +107,16 @@ eaintegration: {% set integration_keys = salt['pillar.get']('elasticfleet:optional_integrations', {}).keys() %} fleet_server_integrations_{{ minion }}: file.directory: - - name: /opt/so/conf/elastic-fleet/integrations/FleetServer_{{ minion }} + - name: /opt/so/conf/elastic-fleet/integrations-optional/FleetServer_{{ minion }} - user: 947 - group: 939 - makedirs: True {% for integration in integration_keys %} {% set enabled_nodes = optional_integrations[integration]["enabled_nodes"] %} {% if minion in enabled_nodes %} -optional_integrations_dynamic_{{ minion }}: +optional_integrations_dynamic_{{ minion }}_{{ integration }}: file.managed: - - name: /opt/so/conf/elastic-fleet/integrations/FleetServer_{{ minion }}/{{ integration }}.json + - name: /opt/so/conf/elastic-fleet/integrations-optional/FleetServer_{{ minion }}/{{ integration }}.json - source: salt://elasticfleet/files/integrations-optional/{{ integration }}.json - user: 947 - group: 939 @@ -120,17 +127,13 @@ optional_integrations_dynamic_{{ minion }}: {% endfor %} {% endif %} {% endfor %} - ea-integrations-load: file.absent: - name: /opt/so/state/eaintegrations.txt - onchanges: - file: eaintegration - file: eadynamicintegration - - file: eapackageupgrade - {% for minion in node_data %} - - file: optional_integrations_dynamic_{{ minion }} - {% endfor %} + - file: /opt/so/conf/elastic-fleet/integrations-optional/* {% endif %} {% else %} From a0926b7b872ea091475e71b6476c3732f99256cd Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 1 Nov 2023 13:59:24 +0000 Subject: [PATCH 07/11] Load optional integrations --- .../so-elastic-fleet-integration-policy-load | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load index 44e7ccf2b..518d29d26 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load @@ -64,8 +64,28 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then if [[ "$RETURN_CODE" != "1" ]]; then touch /opt/so/state/eaintegrations.txt fi + + # Fleet Server - Optional integrations + for INTEGRATION in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json + do + if ! [ "$INTEGRATION" == "/opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json" ]; then + FLEET_POLICY=`echo "$INTEGRATION"| cut -d'/' -f7` + printf "\n\nFleet Server Policy - Loading $INTEGRATION\n" + elastic_fleet_integration_check "$FLEET_POLICY" "$INTEGRATION" + if [ -n "$INTEGRATION_ID" ]; then + printf "\n\nIntegration $NAME exists - Updating integration\n" + elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION" + else + printf "\n\nIntegration does not exist - Creating integration\n" + if [ "$NAME" != "elasticsearch-logs" ]; then + elastic_fleet_integration_create "@$INTEGRATION" + fi + fi + fi + done + if [[ "$RETURN_CODE" != "1" ]]; then + touch /opt/so/state/eaintegrations.txt + fi else exit $RETURN_CODE fi - - From bca1194a468362f36749cbaeafcbb2f15c98b5b4 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 1 Nov 2023 14:01:55 +0000 Subject: [PATCH 08/11] Sublime SOC Action --- salt/soc/defaults.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 6d8ed5bfd..ceca9ef31 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -59,6 +59,12 @@ soc: target: _blank links: - 'https://www.virustotal.com/gui/search/{value}' + - name: Sublime Platform Email Review + description: Review email in Sublime Platform + icon: fa-external-link-alt + target: _blank + links: + - 'https://{:sublime.url}/messages/{:sublime.message_group_id}' eventFields: default: - soc_timestamp From 655c88cd09874abe92106b3df38a8ae0225b4a55 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 1 Nov 2023 16:47:51 +0000 Subject: [PATCH 09/11] Make sure enabled_nodes is populated --- salt/elasticfleet/config.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/elasticfleet/config.sls b/salt/elasticfleet/config.sls index 78fb1d412..434b6db2d 100644 --- a/salt/elasticfleet/config.sls +++ b/salt/elasticfleet/config.sls @@ -112,6 +112,7 @@ fleet_server_integrations_{{ minion }}: - group: 939 - makedirs: True {% for integration in integration_keys %} +{% if 'enabled_nodes' in optional_integrations[integration]%} {% set enabled_nodes = optional_integrations[integration]["enabled_nodes"] %} {% if minion in enabled_nodes %} optional_integrations_dynamic_{{ minion }}_{{ integration }}: @@ -124,6 +125,7 @@ optional_integrations_dynamic_{{ minion }}_{{ integration }}: - defaults: NAME: {{ minion }} {% endif %} +{% endif %} {% endfor %} {% endif %} {% endfor %} From c32935e2e63fbf6e01bd7417e8afdf9453163993 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 1 Nov 2023 17:02:43 +0000 Subject: [PATCH 10/11] Remove optional integration from configuration if not enabled --- salt/elasticfleet/config.sls | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/elasticfleet/config.sls b/salt/elasticfleet/config.sls index 434b6db2d..02672d58f 100644 --- a/salt/elasticfleet/config.sls +++ b/salt/elasticfleet/config.sls @@ -124,6 +124,10 @@ optional_integrations_dynamic_{{ minion }}_{{ integration }}: - template: jinja - defaults: NAME: {{ minion }} +{% else %} +optional_integrations_dynamic_{{ minion }}_{{ integration }}_delete: + file.absent: + - name: /opt/so/conf/elastic-fleet/integrations-optional/FleetServer_{{ minion }}/{{ integration }}.json {% endif %} {% endif %} {% endfor %} From f33079f1e35792a31396f5064aa02dcbf690c2b7 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 1 Nov 2023 20:09:56 +0000 Subject: [PATCH 11/11] Make settings global --- salt/elasticfleet/soc_elasticfleet.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/salt/elasticfleet/soc_elasticfleet.yaml b/salt/elasticfleet/soc_elasticfleet.yaml index 8cb975086..8685a96e5 100644 --- a/salt/elasticfleet/soc_elasticfleet.yaml +++ b/salt/elasticfleet/soc_elasticfleet.yaml @@ -43,32 +43,33 @@ elasticfleet: optional_integrations: sublime_platform: enabled_nodes: - description: Determines if the Sublime Platform integration is enabled. + description: Fleet nodes with the Sublime Platform integration enabled. Enter one per line. global: True helpLink: elastic-fleet.html advanced: True forcedType: "[]string" api_key: description: API key for Sublime Platform. - global: False + global: True helpLink: elastic-fleet.html advanced: True forcedType: string + sensitive: True base_url: description: Base URL for Sublime Platform. - global: False + global: True helpLink: elastic-fleet.html advanced: True forcedType: string poll_interval: description: Poll interval for alerts from Sublime Platform. - global: False + global: True helpLink: elastic-fleet.html advanced: True forcedType: string limit: description: The maximum number of message groups to return from Sublime Platform. - global: False + global: True helpLink: elastic-fleet.html advanced: True forcedType: int