Merge pull request #10462 from Security-Onion-Solutions/feature/elastic_agent_zeek_logging

Dynamic integration configuration and Zeek log exclusions for Elastic Agent
2025-12-23 01:13:09 +01:00 · 2023-05-30 19:27:13 -04:00
parent 4d4744a89b f3be63051b
commit 36791665f3
21 changed files with 197 additions and 75 deletions
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-agent-policy-delete
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-agent-policy-delete
@@ -4,7 +4,7 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

-. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common

 POLICY_ID=$1

--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-agent-policy-list
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-agent-policy-list
@@ -4,14 +4,12 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

-. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common

 # Let's snag a cookie from Kibana
 SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')

-echo "Setting up default Security Onion package policies for Elastic Agent..."
-
 # List configured agent policies
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies" | jq 
+curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies" | jq

 echo
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-agent-policy-view
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-agent-policy-view
@@ -4,16 +4,14 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

-. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common

 POLICY_ID=$1

 # Let's snag a cookie from Kibana
-SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
-
-echo "Viewing agent policy $POLICY_ID"
+SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')

 # View agent policy
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$POLICY_ID/full" | jq
+curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -H "kbn-xsrf: true" -L -X GET "localhost:5601/api/fleet/agent_policies/$POLICY_ID" | jq

 echo
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
@@ -0,0 +1,79 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+DEFAULT_SALT_DIR=/opt/so/saltstack/default
+
+if [ -z $NOROOT ]; then
+        # Check for prerequisites
+        if [ "$(id -u)" -ne 0 ]; then
+                echo "This script must be run using sudo!"
+                exit 1
+        fi
+fi
+
+# Ensure /usr/sbin is in path
+if ! echo "$PATH" | grep -q "/usr/sbin"; then
+  export PATH="$PATH:/usr/sbin"
+fi
+
+# Define a banner to separate sections
+banner="========================================================================="
+
+elastic_fleet_integration_check() {
+
+    AGENT_POLICY=$1
+
+    JSON_STRING=$2
+
+    NAME=$(jq -r .name $JSON_STRING)
+
+    INTEGRATION_ID=$(/usr/sbin/so-elastic-fleet-agent-policy-view "$AGENT_POLICY" | jq -r '.item.package_policies[] | select(.name=="'"$NAME"'") | .id')
+
+}
+
+elastic_fleet_integration_create() {
+
+    JSON_STRING=$1
+
+    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+}
+
+elastic_fleet_integration_update() {
+
+    UPDATE_ID=$1
+
+    JSON_STRING=$2
+
+    curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/package_policies/$UPDATE_ID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+}
+
+elastic_fleet_policy_create() {
+
+    NAME=$1
+    DESC=$2
+    FLEETSERVER=$3
+        TIMEOUT=$4
+
+    JSON_STRING=$( jq -n \
+                    --arg NAME "$NAME" \
+                    --arg DESC "$DESC" \
+                    --arg TIMEOUT $TIMEOUT \
+                    --arg FLEETSERVER "$FLEETSERVER" \
+                    '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":$TIMEOUT,"has_fleet_server":$FLEETSERVER}'
+                    )
+    # Create Fleet Policy
+    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+
+}
+
+elastic_fleet_policy_update() {
+
+    POLICYID=$1
+    JSON_STRING=$2
+
+    curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/$POLICYID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+}
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-data-streams-list
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-data-streams-list
@@ -4,7 +4,7 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

-. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common

 # Let's snag a cookie from Kibana
 SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete
@@ -4,7 +4,7 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

-. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common

 POLICY_ID=$1

--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-delete
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-delete
@@ -4,7 +4,7 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

-. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common

 POLICY_ID=$1

--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-list
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-list
@@ -4,14 +4,12 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

-. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common

 # Let's snag a cookie from Kibana
-SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
-
-echo "Setting up default Security Onion package policies for Elastic Agent..."
+SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')

 # List configured package policies
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/package_policies" | jq 
+curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' | jq

 echo
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load
@@ -4,18 +4,46 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

-. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common

-# Initial Endpoints
-for INTEGRATION in /opt/so/saltstack/default/salt/elasticfleet/files/integrations/endpoints-initial/*.json
-do
-  printf "\n\nInitial Endpoint Policy - Loading $INTEGRATION\n"
-  elastic_fleet_integration_create "@$INTEGRATION"
-done
+RETURN_CODE=0
+
+if [ ! -f /opt/so/state/eaintegrations.txt ]; then
+  # Initial Endpoints
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/endpoints-initial/*.json
+  do
+    printf "\n\nInitial Endpoints Policy - Loading $INTEGRATION\n"
+    elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"
+    if [ -n "$INTEGRATION_ID" ]; then
+      if [ "$NAME" != "elastic-defend-endpoints" ]; then
+        printf "\n\nIntegration $NAME exists - Updating integration\n"
+        elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"
+      fi
+    else
+      printf "\n\nIntegration does not exist - Creating integration\n"
+      elastic_fleet_integration_create "@$INTEGRATION"
+    fi
+  done
+
+  # Grid Nodes
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes/*.json
+  do
+    printf "\n\nGrid Nodes Policy - Loading $INTEGRATION\n"
+    elastic_fleet_integration_check "so-grid-nodes" "$INTEGRATION"
+    if [ -n "$INTEGRATION_ID" ]; then
+      printf "\n\nIntegration $NAME exists - Updating integration\n"
+      elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"
+    else
+      printf "\n\nIntegration does not exist - Creating integration\n"
+      if [ "$NAME" != "elasticsearch-logs" ]; then
+        elastic_fleet_integration_create "@$INTEGRATION"
+      fi
+    fi
+  done
+  if [[ "$RETURN_CODE" != "1" ]]; then
+    touch /opt/so/state/eaintegrations.txt
+  fi
+else
+  exit $RETURN_CODE
+fi

-# Grid Nodes
-for INTEGRATION in /opt/so/saltstack/default/salt/elasticfleet/files/integrations/grid-nodes/*.json
-do
-  printf "\n\nGrid Nodes Policy - Loading $INTEGRATION\n"
-  elastic_fleet_integration_create "@$INTEGRATION"
-done
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-restart
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-restart
@@ -7,6 +7,6 @@



-. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common

 /usr/sbin/so-restart elastic-fleet $1
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-start
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-start
@@ -7,6 +7,6 @@



-. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common

 /usr/sbin/so-start elastic-fleet $1
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-stop
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-stop
@@ -7,6 +7,6 @@



-. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common

 /usr/sbin/so-stop elastic-fleet $1
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers
@@ -8,7 +8,7 @@

 {% from 'vars/globals.map.jinja' import GLOBALS %}

-. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common

 for i in {1..30}
 do
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
@@ -6,7 +6,7 @@
 # this file except in compliance with the Elastic License 2.0.
 {% from 'vars/globals.map.jinja' import GLOBALS %}

-. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common

 printf "\n### Create ES Token ###\n"
 ESTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/service_tokens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' |  jq -r .value)