Automatically manage Fleet Logstash Config

salt/elasticfleet/defaults.yaml

@@ -2,10 +2,11 @@ elasticfleet:
   enabled: False
   config:
     server:
+      custom_fqdn: ''
+      enable_auto_configuration: True
       endpoints_enrollment: ''
       es_token: ''
       grid_enrollment: ''
-      url: ''
   logging:
     zeek:
       excluded:

salt/elasticfleet/enabled.sls

@@ -9,11 +9,22 @@
 {%   from 'docker/docker.map.jinja' import DOCKER %}
 {#   This value is generated during node install and stored in minion pillar #}
 {%   set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %}
+{%   set ENABLEAUTOCONFIGURATION = salt['pillar.get']('elasticfleet:config:server:enable_auto_configuration','') %}
 
 include:
   - elasticfleet.config
   - elasticfleet.sostatus
 
+{% if ENABLEAUTOCONFIGURATION %}
+so-elastic-fleet-auto-configure-logstash-outputs:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-outputs-update
+
+#so-elastic-fleet-auto-configure-server-urls:
+#  cmd.run:
+#    - name: /usr/sbin/so-elastic-fleet-urls-update
+{% endif %}
+
 {%   if SERVICETOKEN != '' %}
 so-elastic-fleet:
   docker_container.running:

salt/elasticfleet/soc_elasticfleet.yaml

@@ -11,6 +11,16 @@ elasticfleet:
         helpLink: zeek.html 
   config:
     server:
+      custom_fqdn:
+        description: Custom FQDN for Agents to connect to.
+        global: True
+        helpLink: elastic-fleet.html
+        advanced: True
+      enable_auto_configuration:
+        description: Enable auto-configuration of Logstash Outputs & Fleet Host URLs.
+        global: True
+        helpLink: elastic-fleet.html
+        advanced: True
       endpoints_enrollment:
         description: Endpoint enrollment key.
         global: True
@@ -29,8 +39,3 @@ elasticfleet:
         helpLink: elastic-fleet.html
         sensitive: True
         advanced: True
-      url:
-        description: Agent connection URL.
-        global: True
-        helpLink: elastic-fleet.html
-        advanced: True

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update

@@ -0,0 +1,75 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
+# this file except in compliance with the Elastic License 2.0.
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+{% set CUSTOMFQDN = salt['pillar.get']('elasticfleet:config:server:custom_fqdn') %}
+
+function update_logstash_outputs() {
+	# Generate updated JSON payload
+    JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":""}')
+
+    # Update Logstash Outputs
+    curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_logstash" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" | jq
+}
+
+# Get current list of Logstash Outputs
+RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_logstash')
+
+# Check to make sure that the server responded with good data - else, bail from script
+CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON")
+if [ "$CHECKSUM" != "so-manager_logstash" ]; then
+ printf "Failed to query for current Logstash Outputs..."
+ exit 1
+fi
+
+# Get the current list of Logstash outputs & hash them
+CURRENT_LIST=$(jq -c -r '.item.hosts' <<<  "$RAW_JSON")
+CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}')
+
+# Create array & add initial elements
+if [ "{{ GLOBALS.manager_ip }}" = "{{ GLOBALS.url_base }}" ]; then
+    NEW_LIST=("{{ GLOBALS.url_base }}:5055")
+else
+    NEW_LIST=("{{ GLOBALS.url_base }}:5055" "{{ GLOBALS.manager_ip }}:5055")
+fi
+
+{% if CUSTOMFQDN != "" %}
+# Add Custom Hostname to list
+NEW_LIST+=("{{ CUSTOMFQDN }}:5055")
+{% endif %}
+
+# Query for the current Grid Nodes that are running Logstash
+LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local')
+
+# Query for Receiver Nodes & add them to the list
+if grep -q "receiver" <<< $LOGSTASHNODES; then
+   readarray -t RECEIVERNODES < <(jq -r ' .receiver | keys_unsorted[]'  <<< $LOGSTASHNODES)
+   for NODE in "${RECEIVERNODES[@]}"
+   do
+    NEW_LIST+=("$NODE:5055")
+   done
+fi
+
+# Query for Fleet Nodes & add them to the list
+if grep -q "fleet" <<< $LOGSTASHNODES; then
+   readarray -t FLEETNODES < <(jq -r ' .fleet | keys_unsorted[]'  <<< $LOGSTASHNODES)
+   for NODE in "${FLEETNODES[@]}"
+   do
+    NEW_LIST+=("$NODE:5055")
+   done
+fi
+
+# Sort & hash the new list of Logstash Outputs
+NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}")
+NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
+
+# Compare the current & new list of outputs - if different, update the Logstash outputs
+if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then
+    printf "\nHashes match - no update needed.\n"
+    printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n"
+    exit 0
+else
+    printf "\nHashes don't match - update needed.\n"
+    printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n"
+    update_logstash_outputs
+fi

salt/manager/tools/sbin/so-minion

@@ -515,7 +515,7 @@ function createFLEET() {
 	add_logstash_to_minion
 	create_fleet_policy
 	update_fleet_host_urls
-	update_logstash_outputs
+	#update_logstash_outputs
 	add_telegraf_to_minion
 	add_nginx_to_minion
 }