Merge pull request #10462 from Security-Onion-Solutions/feature/elastic_agent_zeek_logging

Dynamic integration configuration and Zeek log exclusions for Elastic Agent

salt/common/tools/sbin/so-common

@@ -160,41 +160,6 @@ disable_fastestmirror() {
 	sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf
 }
 
-elastic_fleet_integration_create() {
-
-    JSON_STRING=$1
-
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
-}
-
-elastic_fleet_policy_create() {
-
-    NAME=$1
-    DESC=$2
-    FLEETSERVER=$3
-	TIMEOUT=$4
-
-    JSON_STRING=$( jq -n \
-                    --arg NAME "$NAME" \
-                    --arg DESC "$DESC" \
-                    --arg TIMEOUT $TIMEOUT \
-                    --arg FLEETSERVER "$FLEETSERVER" \
-                    '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":$TIMEOUT,"has_fleet_server":$FLEETSERVER}'
-                    )
-    # Create Fleet Policy
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" 
-
-}
-
-elastic_fleet_policy_update() {
-
-    POLICYID=$1
-    JSON_STRING=$2
-
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/$POLICYID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
-}
-
-
 elastic_license() {
 
 read -r -d '' message <<- EOM

salt/elasticfleet/config.sls

@@ -51,6 +51,35 @@ eastatedir:
     - group: 939
     - makedirs: True
 
+eaintegrationsdir:
+  file.directory:
+    - name: /opt/so/conf/elastic-fleet/integrations
+    - user: 947
+    - group: 939
+    - makedirs: True
+
+eadynamicintegration:
+  file.recurse:
+    - name: /opt/so/conf/elastic-fleet/integrations
+    - source: salt://elasticfleet/files/integrations-dynamic
+    - user: 947
+    - group: 939
+    - template: jinja
+
+eaintegration:
+  file.recurse:
+    - name: /opt/so/conf/elastic-fleet/integrations
+    - source: salt://elasticfleet/files/integrations
+    - user: 947
+    - group: 939
+
+ea-integrations-load:
+  file.absent:
+    - name: /opt/so/state/eaintegrations.txt
+    - onchanges:
+      - file: eaintegration
+      - file: eadynamicintegration
+
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/elasticfleet/enabled.sls

@@ -62,6 +62,10 @@ so-elastic-fleet:
       {% endif %}
 {%   endif %}
 
+so-elastic-fleet-integrations:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-integration-policy-load
+
 delete_so-elastic-fleet_so-status.disabled:
   file.uncomment:
     - name: /opt/so/conf/so-status/so-status.conf

salt/elasticfleet/files/integrations-dynamic/grid-nodes/import-zeek-logs.json

@@ -1,3 +1,5 @@
+{% from 'zeek/config.map.jinja' import ZEEKMERGED %}
+{%- raw -%}
 {
   "package": {
     "name": "log",
@@ -20,10 +22,11 @@
             "data_stream.dataset": "import",
             "tags": [],
             "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/zeek/logs/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- script:\n      lang: javascript\n      source: >\n        function process(event) {\n          var pl = event.Get(\"import.file\").slice(0,-4);\n          event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n        }\n- add_fields:\n    target: event\n    fields:\n      category: network\n      module: zeek\n      imported: true\n- add_tags:\n    tags: \"ics\"\n    when:\n      regexp:\n        import.file: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"",
-            "custom": "exclude_files: [\"broker|capture_loss|ecat_arp_info|loaded_scripts|packet_filter|stats|stderr|stdout.log$\"]\n"
+            "custom": "exclude_files: [\"{%- endraw -%}{{ ZEEKMERGED.logging.excluded | join('|') }}{%- raw -%}.log$\"]\n"
           }
         }
       }
     }
   }
 }
+{%- endraw -%}

salt/elasticfleet/files/integrations-dynamic/grid-nodes/zeek-logs.json

@@ -1,8 +1,11 @@
+{% from 'zeek/config.map.jinja' import ZEEKMERGED %}
+{%- raw -%}
 {
   "package": {
     "name": "log",
     "version": ""
   },
+  "id": "zeek-logs",
   "name": "zeek-logs",
   "namespace": "so",
   "description": "Zeek logs",
@@ -20,10 +23,11 @@
             "data_stream.dataset": "zeek",
             "tags": [],
             "processors": "- dissect:\n    tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n    field: \"log.file.path\"\n    trim_chars: \".log\"\n    target_prefix: \"\"\n- script:\n      lang: javascript\n      source: >\n        function process(event) {\n          var pl = event.Get(\"pipeline\");\n          event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n        }\n- add_fields:\n    target: event\n    fields:\n      category: network\n      module: zeek\n- add_tags:\n    tags: \"ics\"\n    when:\n      regexp:\n        pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"",
-            "custom": "exclude_files: [\"broker|capture_loss|ecat_arp_info|known_hosts|known_services|loaded_scripts|ntp|packet_filter|reporter|stats|stderr|stdout.log$\"]\n"
+            "custom": "exclude_files: [\"{%- endraw -%}{{ ZEEKMERGED.logging.excluded | join('|') }}{%- raw -%}.log$\"]\n"
           }
         }
       }
     }
   }
 }
+{%- endraw -%}

salt/elasticfleet/tools/sbin/so-elastic-fleet-agent-policy-delete

@@ -4,7 +4,7 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common
 
 POLICY_ID=$1
 

salt/elasticfleet/tools/sbin/so-elastic-fleet-agent-policy-list

@@ -4,14 +4,12 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common
 
 # Let's snag a cookie from Kibana
 SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
 
-echo "Setting up default Security Onion package policies for Elastic Agent..."
-
 # List configured agent policies
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies" | jq 
+curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies" | jq
 
 echo

salt/elasticfleet/tools/sbin/so-elastic-fleet-agent-policy-view

@@ -4,16 +4,14 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common
 
 POLICY_ID=$1
 
 # Let's snag a cookie from Kibana
-SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
-
-echo "Viewing agent policy $POLICY_ID"
 
 # View agent policy
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$POLICY_ID/full" | jq
+curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -H "kbn-xsrf: true" -L -X GET "localhost:5601/api/fleet/agent_policies/$POLICY_ID" | jq
 
 echo

salt/elasticfleet/tools/sbin/so-elastic-fleet-common

@@ -0,0 +1,79 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+DEFAULT_SALT_DIR=/opt/so/saltstack/default
+
+if [ -z $NOROOT ]; then
+        # Check for prerequisites
+        if [ "$(id -u)" -ne 0 ]; then
+                echo "This script must be run using sudo!"
+                exit 1
+        fi
+fi
+
+# Ensure /usr/sbin is in path
+if ! echo "$PATH" | grep -q "/usr/sbin"; then
+  export PATH="$PATH:/usr/sbin"
+fi
+
+# Define a banner to separate sections
+banner="========================================================================="
+
+elastic_fleet_integration_check() {
+
+    AGENT_POLICY=$1
+
+    JSON_STRING=$2
+
+    NAME=$(jq -r .name $JSON_STRING)
+
+    INTEGRATION_ID=$(/usr/sbin/so-elastic-fleet-agent-policy-view "$AGENT_POLICY" | jq -r '.item.package_policies[] | select(.name=="'"$NAME"'") | .id')
+
+}
+
+elastic_fleet_integration_create() {
+
+    JSON_STRING=$1
+
+    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+}
+
+elastic_fleet_integration_update() {
+
+    UPDATE_ID=$1
+
+    JSON_STRING=$2
+
+    curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/package_policies/$UPDATE_ID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+}
+
+elastic_fleet_policy_create() {
+
+    NAME=$1
+    DESC=$2
+    FLEETSERVER=$3
+        TIMEOUT=$4
+
+    JSON_STRING=$( jq -n \
+                    --arg NAME "$NAME" \
+                    --arg DESC "$DESC" \
+                    --arg TIMEOUT $TIMEOUT \
+                    --arg FLEETSERVER "$FLEETSERVER" \
+                    '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":$TIMEOUT,"has_fleet_server":$FLEETSERVER}'
+                    )
+    # Create Fleet Policy
+    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+
+}
+
+elastic_fleet_policy_update() {
+
+    POLICYID=$1
+    JSON_STRING=$2
+
+    curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/$POLICYID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+}

salt/elasticfleet/tools/sbin/so-elastic-fleet-data-streams-list

@@ -4,7 +4,7 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common
 
 # Let's snag a cookie from Kibana
 SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete

@@ -4,7 +4,7 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common
 
 POLICY_ID=$1
 

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-delete

@@ -4,7 +4,7 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common
 
 POLICY_ID=$1
 

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-list

@@ -4,14 +4,12 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common
 
 # Let's snag a cookie from Kibana
-SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
-
-echo "Setting up default Security Onion package policies for Elastic Agent..."
 
 # List configured package policies
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/package_policies" | jq 
+curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' | jq
 
 echo

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -4,18 +4,46 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common
 
-# Initial Endpoints
+RETURN_CODE=0
-for INTEGRATION in /opt/so/saltstack/default/salt/elasticfleet/files/integrations/endpoints-initial/*.json
+
-do
+if [ ! -f /opt/so/state/eaintegrations.txt ]; then
-  printf "\n\nInitial Endpoint Policy - Loading $INTEGRATION\n"
+  # Initial Endpoints
-  elastic_fleet_integration_create "@$INTEGRATION"
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/endpoints-initial/*.json
-done
+  do
+    printf "\n\nInitial Endpoints Policy - Loading $INTEGRATION\n"
+    elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"
+    if [ -n "$INTEGRATION_ID" ]; then
+      if [ "$NAME" != "elastic-defend-endpoints" ]; then
+        printf "\n\nIntegration $NAME exists - Updating integration\n"
+        elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"
+      fi
+    else
+      printf "\n\nIntegration does not exist - Creating integration\n"
+      elastic_fleet_integration_create "@$INTEGRATION"
+    fi
+  done
+
+  # Grid Nodes
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes/*.json
+  do
+    printf "\n\nGrid Nodes Policy - Loading $INTEGRATION\n"
+    elastic_fleet_integration_check "so-grid-nodes" "$INTEGRATION"
+    if [ -n "$INTEGRATION_ID" ]; then
+      printf "\n\nIntegration $NAME exists - Updating integration\n"
+      elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"
+    else
+      printf "\n\nIntegration does not exist - Creating integration\n"
+      if [ "$NAME" != "elasticsearch-logs" ]; then
+        elastic_fleet_integration_create "@$INTEGRATION"
+      fi
+    fi
+  done
+  if [[ "$RETURN_CODE" != "1" ]]; then
+    touch /opt/so/state/eaintegrations.txt
+  fi
+else
+  exit $RETURN_CODE
+fi
 
-# Grid Nodes
-for INTEGRATION in /opt/so/saltstack/default/salt/elasticfleet/files/integrations/grid-nodes/*.json
-do
-  printf "\n\nGrid Nodes Policy - Loading $INTEGRATION\n"
-  elastic_fleet_integration_create "@$INTEGRATION"
-done

salt/elasticfleet/tools/sbin/so-elastic-fleet-restart

@@ -7,6 +7,6 @@
 
 
 
-. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common
 
 /usr/sbin/so-restart elastic-fleet $1

salt/elasticfleet/tools/sbin/so-elastic-fleet-start

@@ -7,6 +7,6 @@
 
 
 
-. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common
 
 /usr/sbin/so-start elastic-fleet $1

salt/elasticfleet/tools/sbin/so-elastic-fleet-stop

@@ -7,6 +7,6 @@
 
 
 
-. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common
 
 /usr/sbin/so-stop elastic-fleet $1

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers

@@ -8,7 +8,7 @@
 
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 
-. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common
 
 for i in {1..30}
 do

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup

@@ -6,7 +6,7 @@
 # this file except in compliance with the Elastic License 2.0.
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 
-. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common
 
 printf "\n### Create ES Token ###\n"
 ESTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/service_tokens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' |  jq -r .value)

salt/zeek/defaults.yaml

@@ -107,3 +107,18 @@ zeek:
     - application/vnd.ms-powerpoint.presentation.macroenabled.12: doc
     - application/vnd.ms-powerpoint.slideshow.macroenabled.12: doc
     - application/vnd.openxmlformats-officedocument: doc
+  logging:
+    excluded:
+      - broker
+      - capture_loss
+      - ecat_arp_info
+      - known_hosts
+      - known_services
+      - loaded_scripts
+      - ntp
+      - packet_filter
+      - reporter
+      - stats
+      - stderr
+      - stdout
+ 

salt/zeek/soc_zeek.yaml

@@ -3,8 +3,9 @@ zeek:
     description: You can enable or disable ZEEK on all sensors or a single sensor.
     helpLink: zeek.html
   logging:
-    enabled:
+    excluded:
-      description: This is a list of Zeek logs that will be shipped through the pipeline. If you remove a log from this list, it will still persist on the sensor.
+      description: This is a list of Zeek logs that are excluded from being shipped through the data processing pipeline. If you remove a log from this list, Elastic Agent will attempt to process it. If an ingest node pipeline is not available to process the logs, you may experience errors.
+      forcedType: "[]string"
       helpLink: zeek.html
   config:
     local: