diff --git a/salt/elasticfleet/config.sls b/salt/elasticfleet/config.sls index d2e357c91..02672d58f 100644 --- a/salt/elasticfleet/config.sls +++ b/salt/elasticfleet/config.sls @@ -6,6 +6,7 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} {% if sls.split('.')[0] in allowed_states %} +{% set node_data = salt['pillar.get']('node_data') %} # Add EA Group elasticfleetgroup: @@ -92,13 +93,53 @@ eaintegration: - user: 947 - group: 939 +eaoptionalintegrationsdir: + file.directory: + - name: /opt/so/conf/elastic-fleet/integrations-optional + - user: 947 + - group: 939 + - makedirs: True + +{% for minion in node_data %} +{% set role = node_data[minion]["role"] %} +{% if role in [ "fleet","heavynode", "manager","managersearch","standalone" ] %} +{% set optional_integrations = salt['pillar.get']('elasticfleet:optional_integrations', {}) %} +{% set integration_keys = salt['pillar.get']('elasticfleet:optional_integrations', {}).keys() %} +fleet_server_integrations_{{ minion }}: + file.directory: + - name: /opt/so/conf/elastic-fleet/integrations-optional/FleetServer_{{ minion }} + - user: 947 + - group: 939 + - makedirs: True +{% for integration in integration_keys %} +{% if 'enabled_nodes' in optional_integrations[integration]%} +{% set enabled_nodes = optional_integrations[integration]["enabled_nodes"] %} +{% if minion in enabled_nodes %} +optional_integrations_dynamic_{{ minion }}_{{ integration }}: + file.managed: + - name: /opt/so/conf/elastic-fleet/integrations-optional/FleetServer_{{ minion }}/{{ integration }}.json + - source: salt://elasticfleet/files/integrations-optional/{{ integration }}.json + - user: 947 + - group: 939 + - template: jinja + - defaults: + NAME: {{ minion }} +{% else %} +optional_integrations_dynamic_{{ minion }}_{{ integration }}_delete: + file.absent: + - name: /opt/so/conf/elastic-fleet/integrations-optional/FleetServer_{{ minion }}/{{ integration }}.json +{% endif %} +{% endif %} +{% endfor %} +{% endif %} +{% endfor %} ea-integrations-load: file.absent: - name: /opt/so/state/eaintegrations.txt - onchanges: - file: eaintegration - file: eadynamicintegration - - file: eapackageupgrade + - file: /opt/so/conf/elastic-fleet/integrations-optional/* {% endif %} {% else %} diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index a17957e7c..ba779f5a0 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -90,3 +90,10 @@ elasticfleet: - zscaler_zia - zscaler_zpa - 1password + optional_integrations: + sublime_platform: + enabled_nodes: [] + api_key: + base_url: https://api.platform.sublimesecurity.com + poll_interval: 5m + limit: 100 diff --git a/salt/elasticfleet/files/integrations-optional/sublime_platform.json b/salt/elasticfleet/files/integrations-optional/sublime_platform.json new file mode 100644 index 000000000..8feedc879 --- /dev/null +++ b/salt/elasticfleet/files/integrations-optional/sublime_platform.json @@ -0,0 +1,44 @@ +{%- from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED -%} +{%- from 'sensoroni/map.jinja' import SENSORONIMERGED -%} +{%- from 'vars/globals.map.jinja' import GLOBALS -%} +{%- raw -%} +{ + "package": { + "name": "httpjson", + "version": "" + }, + "name": "sublime-platform", + "namespace": "default", + "description": "", + "policy_id": "FleetServer_{%- endraw -%}{{ NAME }}{%- raw -%}", + "vars": {}, + "inputs": { + "generic-httpjson": { + "enabled": true, + "streams": { + "httpjson.generic": { + "enabled": true, + "vars": { + "request_method": "GET", + "processors": "- drop_event:\n when:\n not:\n contains: \n message: \"flagged_rules\"\n- decode_json_fields:\n fields: [\"message\"]\n document_id: id\n target: \"\"", + "enable_request_tracer": false, + "oauth_scopes": [], + "request_transforms": "- set:\n target: header.Authorization\n value: 'Bearer {% endraw -%}{{ ELASTICFLEETMERGED.optional_integrations.sublime_platform.api_key }}{%- raw -%}'\n- set:\n target: header.accept\n value: application/json\n- set:\n target: url.params.last_message_created_at[gte]\n value: '[[formatDate (now (parseDuration \"-{%- endraw -%}{{ ELASTICFLEETMERGED.optional_integrations.sublime_platform.poll_interval }}{%- raw -%}\")) \"2006-01-02T15:04:05Z\"]]'\n- set:\n target: url.params.reviewed\n value: false\n- set:\n target: url.params.flagged\n value: true\n- set:\n target: url.params.limit\n value: {% endraw %}{{ ELASTICFLEETMERGED.optional_integrations.sublime_platform.limit }}{%- raw -%}", + "response_transforms": "", + "request_redirect_headers_ban_list": [], + "request_encode_as": "application/x-www-form-urlencoded", + "request_url": "{%- endraw -%}{{ ELASTICFLEETMERGED.optional_integrations.sublime_platform.base_url }}{%- raw -%}/v0/message-groups", + "response_split": "target: body.message_groups\ntype: array\nkeep_parent: false\ntransforms:\n - set:\n target: body.sublime.request_url\n value : '[[ .last_response.url.value ]]'", + "tags": [ + "forwarded" + ], + "pipeline": "sublime", + "data_stream.dataset": "sublime", + "request_interval": "1m" + } + } + } + } + } +} +{%- endraw -%} diff --git a/salt/elasticfleet/soc_elasticfleet.yaml b/salt/elasticfleet/soc_elasticfleet.yaml index af660358a..8685a96e5 100644 --- a/salt/elasticfleet/soc_elasticfleet.yaml +++ b/salt/elasticfleet/soc_elasticfleet.yaml @@ -40,3 +40,36 @@ elasticfleet: helpLink: elastic-fleet.html sensitive: True advanced: True + optional_integrations: + sublime_platform: + enabled_nodes: + description: Fleet nodes with the Sublime Platform integration enabled. Enter one per line. + global: True + helpLink: elastic-fleet.html + advanced: True + forcedType: "[]string" + api_key: + description: API key for Sublime Platform. + global: True + helpLink: elastic-fleet.html + advanced: True + forcedType: string + sensitive: True + base_url: + description: Base URL for Sublime Platform. + global: True + helpLink: elastic-fleet.html + advanced: True + forcedType: string + poll_interval: + description: Poll interval for alerts from Sublime Platform. + global: True + helpLink: elastic-fleet.html + advanced: True + forcedType: string + limit: + description: The maximum number of message groups to return from Sublime Platform. + global: True + helpLink: elastic-fleet.html + advanced: True + forcedType: int diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load index 44e7ccf2b..518d29d26 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load @@ -64,8 +64,28 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then if [[ "$RETURN_CODE" != "1" ]]; then touch /opt/so/state/eaintegrations.txt fi + + # Fleet Server - Optional integrations + for INTEGRATION in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json + do + if ! [ "$INTEGRATION" == "/opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json" ]; then + FLEET_POLICY=`echo "$INTEGRATION"| cut -d'/' -f7` + printf "\n\nFleet Server Policy - Loading $INTEGRATION\n" + elastic_fleet_integration_check "$FLEET_POLICY" "$INTEGRATION" + if [ -n "$INTEGRATION_ID" ]; then + printf "\n\nIntegration $NAME exists - Updating integration\n" + elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION" + else + printf "\n\nIntegration does not exist - Creating integration\n" + if [ "$NAME" != "elasticsearch-logs" ]; then + elastic_fleet_integration_create "@$INTEGRATION" + fi + fi + fi + done + if [[ "$RETURN_CODE" != "1" ]]; then + touch /opt/so/state/eaintegrations.txt + fi else exit $RETURN_CODE fi - - diff --git a/salt/elasticsearch/files/ingest/sublime b/salt/elasticsearch/files/ingest/sublime new file mode 100644 index 000000000..c26f93c01 --- /dev/null +++ b/salt/elasticsearch/files/ingest/sublime @@ -0,0 +1,34 @@ +{ + "description" : " Email alerts from Sublime", + "processors" : [ + { "set": { "field": "event.module", "value": "sublime" } }, + { "set": { "field": "event.dataset", "value": "alert" } }, + { "set": { "field": "event.severity", "value": 3, "override": true } }, + { "set": { "field": "rule.name", "value": "Sublime Platform: {{ flagged_rules.0.name }}", "override": true } }, + { "set": { "field": "sublime.message_group_id", "value": "{{ _id }}", "override": true } }, + { "set": { "field": "email.address", "value": "{{ messages.0.recipients.0.email }}", "override": true } }, + { "set": { "field": "email.forwarded_recipents", "value": "{{ messages.0.forwarded_receipients }}", "override": true } }, + { "set": { "field": "email.sender.address", "value": "{{ messages.0.sender.email }}", "override": true } }, + { "set": { "field": "email.subject", "value": "{{ messages.0.subject }}", "override": true } }, + { "set": { "field": "email.forwarded_at", "value": "{{ messages.0.forwarded_at }}", "override": true } }, + { "set": { "field": "email.created_at", "value": "{{ messages.0.created_at }}", "override": true } }, + { "set": { "field": "email.read_at", "value": "{{ messages.0.read_at }}", "override": true } }, + { "set": { "field": "email.replied_at", "value": "{{ messages.0.replied_at }}", "override": true } }, + { + "grok": { + "field": "sublime.request_url", + "patterns": ["^https://api.%{DATA:sublime_host}/v0%{GREEDYDATA}$"], + "ignore_failure": true + } + }, + + { "rename": { "field": "sublime_host", "target_field": "sublime.url", "ignore_missing": true } }, + { "rename": { "field": "data", "target_field": "sublime", "ignore_missing": true } }, + { "rename": { "field": "flagged_rules", "target_field": "sublime.flagged_rules", "ignore_missing": true } }, + { "rename": { "field": "organization_id", "target_field": "sublime.organization_id", "ignore_missing": true } }, + { "rename": { "field": "review_status", "target_field": "sublime.review_status", "ignore_missing": true } }, + { "rename": { "field": "state", "target_field": "sublime.state", "ignore_missing": true } }, + { "rename": { "field": "user_reports", "target_field": "sublime.user_reports", "ignore_missing": true } }, + { "pipeline": { "name": "common" } } + ] +} diff --git a/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja b/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja index 0a148155c..f7671e2b7 100644 --- a/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja +++ b/salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja @@ -1,13 +1,16 @@ output { if "elastic-agent" in [tags] { - if [metadata][pipeline] { + if [metadata][pipeline] { + if [metadata][_id] { elasticsearch { hosts => "{{ GLOBALS.manager }}" ecs_compatibility => v8 data_stream => true user => "{{ ES_USER }}" password => "{{ ES_PASS }}" + document_id => "%{[metadata][_id]}" pipeline => "%{[metadata][pipeline]}" + silence_errors_in_log => ["version_conflict_engine_exception"] ssl => true ssl_certificate_verification => false } @@ -19,10 +22,22 @@ output { data_stream => true user => "{{ ES_USER }}" password => "{{ ES_PASS }}" + pipeline => "%{[metadata][pipeline]}" ssl => true - ssl_certificate_verification => false + ssl_certificate_verification => false } - } + } + } + else { + elasticsearch { + hosts => "{{ GLOBALS.manager }}" + ecs_compatibility => v8 + data_stream => true + user => "{{ ES_USER }}" + password => "{{ ES_PASS }}" + ssl => true + ssl_certificate_verification => false + } + } } } - diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 6d8ed5bfd..ceca9ef31 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -59,6 +59,12 @@ soc: target: _blank links: - 'https://www.virustotal.com/gui/search/{value}' + - name: Sublime Platform Email Review + description: Review email in Sublime Platform + icon: fa-external-link-alt + target: _blank + links: + - 'https://{:sublime.url}/messages/{:sublime.message_group_id}' eventFields: default: - soc_timestamp