Merge pull request #11676 from Security-Onion-Solutions/feature/sublime_platform_integration

Sublime Platform Integration

salt/elasticfleet/config.sls

@@ -6,6 +6,7 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% if sls.split('.')[0] in allowed_states %}
+{% set node_data = salt['pillar.get']('node_data') %}
 
 # Add EA Group
 elasticfleetgroup:
@@ -92,13 +93,53 @@ eaintegration:
     - user: 947
     - group: 939
 
+eaoptionalintegrationsdir:
+  file.directory:
+    - name: /opt/so/conf/elastic-fleet/integrations-optional
+    - user: 947
+    - group: 939
+    - makedirs: True
+
+{% for minion in node_data %}
+{% set role = node_data[minion]["role"] %}
+{% if role in [ "fleet","heavynode", "manager","managersearch","standalone" ] %}
+{% set optional_integrations = salt['pillar.get']('elasticfleet:optional_integrations', {}) %}
+{% set integration_keys = salt['pillar.get']('elasticfleet:optional_integrations', {}).keys() %}
+fleet_server_integrations_{{ minion }}:
+  file.directory:
+    - name: /opt/so/conf/elastic-fleet/integrations-optional/FleetServer_{{ minion }}
+    - user: 947
+    - group: 939
+    - makedirs: True
+{% for integration in integration_keys %}
+{% if 'enabled_nodes' in optional_integrations[integration]%}
+{% set enabled_nodes = optional_integrations[integration]["enabled_nodes"] %}
+{% if minion in enabled_nodes %}
+optional_integrations_dynamic_{{ minion }}_{{ integration }}:
+  file.managed:
+    - name: /opt/so/conf/elastic-fleet/integrations-optional/FleetServer_{{ minion }}/{{ integration }}.json
+    - source: salt://elasticfleet/files/integrations-optional/{{ integration }}.json
+    - user: 947
+    - group: 939
+    - template: jinja
+    - defaults:
+        NAME: {{ minion }}
+{% else %}
+optional_integrations_dynamic_{{ minion }}_{{ integration }}_delete:
+  file.absent:
+    - name: /opt/so/conf/elastic-fleet/integrations-optional/FleetServer_{{ minion }}/{{ integration }}.json
+{% endif %}
+{% endif %}
+{% endfor %}
+{% endif %}
+{% endfor %}
 ea-integrations-load:
   file.absent:
     - name: /opt/so/state/eaintegrations.txt
     - onchanges:
       - file: eaintegration
       - file: eadynamicintegration
-      - file: eapackageupgrade
+      - file: /opt/so/conf/elastic-fleet/integrations-optional/*
 {% endif %}
 {% else %}
 

salt/elasticfleet/defaults.yaml

@@ -90,3 +90,10 @@ elasticfleet:
     - zscaler_zia
     - zscaler_zpa
     - 1password
+  optional_integrations:
+    sublime_platform:
+      enabled_nodes: []
+      api_key:
+      base_url: https://api.platform.sublimesecurity.com
+      poll_interval: 5m
+      limit: 100

salt/elasticfleet/files/integrations-optional/sublime_platform.json

@@ -0,0 +1,44 @@
+{%- from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED -%}
+{%- from 'sensoroni/map.jinja' import SENSORONIMERGED -%}
+{%- from 'vars/globals.map.jinja' import GLOBALS -%}
+{%- raw -%}
+{
+  "package": {
+    "name": "httpjson",
+    "version": ""
+  },
+  "name": "sublime-platform",
+  "namespace": "default",
+  "description": "",
+  "policy_id": "FleetServer_{%- endraw -%}{{ NAME }}{%- raw -%}",
+  "vars": {},
+  "inputs": {
+    "generic-httpjson": {
+      "enabled": true,
+      "streams": {
+        "httpjson.generic": {
+          "enabled": true,
+          "vars": {
+            "request_method": "GET",
+            "processors": "- drop_event:\n    when:\n        not:\n            contains: \n                message: \"flagged_rules\"\n- decode_json_fields:\n    fields: [\"message\"]\n    document_id: id\n    target: \"\"",
+            "enable_request_tracer": false,
+            "oauth_scopes": [],
+            "request_transforms": "- set:\n    target: header.Authorization\n    value: 'Bearer {% endraw -%}{{ ELASTICFLEETMERGED.optional_integrations.sublime_platform.api_key }}{%- raw -%}'\n- set:\n    target: header.accept\n    value: application/json\n- set:\n    target: url.params.last_message_created_at[gte]\n    value: '[[formatDate (now (parseDuration \"-{%- endraw -%}{{ ELASTICFLEETMERGED.optional_integrations.sublime_platform.poll_interval }}{%- raw -%}\")) \"2006-01-02T15:04:05Z\"]]'\n- set:\n    target: url.params.reviewed\n    value: false\n- set:\n    target: url.params.flagged\n    value: true\n- set:\n    target: url.params.limit\n    value: {% endraw %}{{ ELASTICFLEETMERGED.optional_integrations.sublime_platform.limit }}{%- raw -%}",
+            "response_transforms": "",
+            "request_redirect_headers_ban_list": [],
+            "request_encode_as": "application/x-www-form-urlencoded",
+            "request_url": "{%- endraw -%}{{ ELASTICFLEETMERGED.optional_integrations.sublime_platform.base_url }}{%- raw -%}/v0/message-groups",
+            "response_split": "target: body.message_groups\ntype: array\nkeep_parent: false\ntransforms:\n    - set:\n        target: body.sublime.request_url\n        value : '[[ .last_response.url.value ]]'",
+            "tags": [
+              "forwarded"
+            ],
+            "pipeline": "sublime",
+            "data_stream.dataset": "sublime",
+            "request_interval": "1m"
+          }
+        }
+      }
+    }
+  }
+}
+{%- endraw -%}

salt/elasticfleet/soc_elasticfleet.yaml

@@ -40,3 +40,36 @@ elasticfleet:
         helpLink: elastic-fleet.html
         sensitive: True
         advanced: True
+  optional_integrations:
+    sublime_platform:
+      enabled_nodes:
+        description: Fleet nodes with the Sublime Platform integration enabled. Enter one per line.
+        global: True
+        helpLink: elastic-fleet.html
+        advanced: True
+        forcedType: "[]string"
+      api_key:
+        description: API key for Sublime Platform.
+        global: True
+        helpLink: elastic-fleet.html
+        advanced: True
+        forcedType: string
+        sensitive: True
+      base_url:
+        description: Base URL for Sublime Platform.
+        global: True
+        helpLink: elastic-fleet.html
+        advanced: True
+        forcedType: string
+      poll_interval:
+        description: Poll interval for alerts from Sublime Platform.
+        global: True
+        helpLink: elastic-fleet.html
+        advanced: True
+        forcedType: string
+      limit:
+        description: The maximum number of message groups to return from Sublime Platform.
+        global: True
+        helpLink: elastic-fleet.html
+        advanced: True
+        forcedType: int

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -64,8 +64,28 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
   if [[ "$RETURN_CODE" != "1" ]]; then
     touch /opt/so/state/eaintegrations.txt
   fi
+
+  # Fleet Server - Optional integrations
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json
+  do
+    if ! [ "$INTEGRATION" == "/opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json" ]; then
+      FLEET_POLICY=`echo "$INTEGRATION"| cut -d'/' -f7`
+      printf "\n\nFleet Server Policy - Loading $INTEGRATION\n"
+      elastic_fleet_integration_check "$FLEET_POLICY" "$INTEGRATION"
+      if [ -n "$INTEGRATION_ID" ]; then
+        printf "\n\nIntegration $NAME exists - Updating integration\n"
+        elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"
+      else
+        printf "\n\nIntegration does not exist - Creating integration\n"
+        if [ "$NAME" != "elasticsearch-logs" ]; then
+          elastic_fleet_integration_create "@$INTEGRATION"
+        fi
+      fi
+    fi
+  done
+  if [[ "$RETURN_CODE" != "1" ]]; then
+    touch /opt/so/state/eaintegrations.txt
+  fi
 else
   exit $RETURN_CODE
 fi
-
-

salt/elasticsearch/files/ingest/sublime

@@ -0,0 +1,34 @@
+{
+  "description" : " Email alerts from Sublime",
+  "processors" : [
+    { "set":         { "field": "event.module", "value": "sublime"  } },
+    { "set":         { "field": "event.dataset", "value": "alert"  } },
+    { "set":         { "field": "event.severity", "value": 3, "override": true }  },
+    { "set":         { "field": "rule.name", "value": "Sublime Platform: {{ flagged_rules.0.name }}", "override": true }  },
+    { "set":         { "field": "sublime.message_group_id", "value": "{{ _id }}", "override": true }  },
+    { "set":         { "field": "email.address", "value": "{{ messages.0.recipients.0.email }}", "override": true }  },
+    { "set":         { "field": "email.forwarded_recipents", "value": "{{ messages.0.forwarded_receipients }}", "override": true }  },
+    { "set":         { "field": "email.sender.address", "value": "{{ messages.0.sender.email }}", "override": true }  },
+    { "set":         { "field": "email.subject", "value": "{{ messages.0.subject }}", "override": true }  },
+    { "set":         { "field": "email.forwarded_at", "value": "{{ messages.0.forwarded_at }}", "override": true }  },
+    { "set":         { "field": "email.created_at", "value": "{{ messages.0.created_at }}", "override": true }  },
+    { "set":         { "field": "email.read_at", "value": "{{ messages.0.read_at }}", "override": true }  },
+    { "set":         { "field": "email.replied_at", "value": "{{ messages.0.replied_at }}", "override": true }  },
+    {
+        "grok": {
+          "field": "sublime.request_url",
+          "patterns": ["^https://api.%{DATA:sublime_host}/v0%{GREEDYDATA}$"],
+          "ignore_failure": true
+        }
+    },
+
+    { "rename":      { "field": "sublime_host", "target_field": "sublime.url", "ignore_missing": true } },
+    { "rename":      { "field": "data", "target_field": "sublime", "ignore_missing": true } },
+    { "rename":      { "field": "flagged_rules", "target_field": "sublime.flagged_rules", "ignore_missing": true } },
+    { "rename":      { "field": "organization_id", "target_field": "sublime.organization_id", "ignore_missing": true } },
+    { "rename":      { "field": "review_status", "target_field": "sublime.review_status", "ignore_missing": true } },
+    { "rename":      { "field": "state", "target_field": "sublime.state", "ignore_missing": true } },
+    { "rename":      { "field": "user_reports", "target_field": "sublime.user_reports", "ignore_missing": true } },
+    { "pipeline":    { "name": "common" } }
+  ]
+}

salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja

@@ -1,6 +1,21 @@
 output {
   if "elastic-agent" in [tags] {
     if [metadata][pipeline] {
+      if [metadata][_id] {
+        elasticsearch {
+          hosts => "{{ GLOBALS.manager }}"
+          ecs_compatibility => v8
+          data_stream => true
+          user => "{{ ES_USER }}"
+          password => "{{ ES_PASS }}"
+          document_id => "%{[metadata][_id]}"
+          pipeline => "%{[metadata][pipeline]}"
+          silence_errors_in_log => ["version_conflict_engine_exception"]
+          ssl => true
+          ssl_certificate_verification => false
+        }
+      }
+      else {
         elasticsearch {
           hosts => "{{ GLOBALS.manager }}"
           ecs_compatibility => v8
@@ -12,6 +27,7 @@ output {
           ssl_certificate_verification => false
         }
       }
+    }
     else {
       elasticsearch {
         hosts => "{{ GLOBALS.manager }}"
@@ -25,4 +41,3 @@ output {
     }
   }
 }
-

salt/soc/defaults.yaml

@@ -59,6 +59,12 @@ soc:
         target: _blank
         links:
           - 'https://www.virustotal.com/gui/search/{value}'
+      - name: Sublime Platform Email Review
+        description: Review email in Sublime Platform
+        icon: fa-external-link-alt
+        target: _blank
+        links:
+          - 'https://{:sublime.url}/messages/{:sublime.message_group_id}' 
     eventFields:
       default:
         - soc_timestamp