Merge pull request #10765 from Security-Onion-Solutions/2.4/FleetEnhancements

Retry install

salt/elasticfleet/enabled.sls

@@ -16,14 +16,25 @@ include:
   - elasticfleet.config
   - elasticfleet.sostatus
 
-{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval'] %}
+# If enabled, automatically update Fleet Logstash Outputs
+{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval', 'so-fleet'] %}
 so-elastic-fleet-auto-configure-logstash-outputs:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-outputs-update
+{% endif %}
 
-#so-elastic-fleet-auto-configure-server-urls:
+# If enabled, automatically update Fleet Server URLs & ES Connection
-#  cmd.run:
+{% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-fleet'] %}
-#    - name: /usr/sbin/so-elastic-fleet-urls-update
+so-elastic-fleet-auto-configure-server-urls:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-urls-update
+{% endif %}
+
+# Automatically update Fleet Server Elasticsearch URLs
+{% if grains.role not in ['so-fleet'] %}
+so-elastic-fleet-auto-configure-elasticsearch-urls:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-es-url-update
 {% endif %}
 
 {%   if SERVICETOKEN != '' %}

salt/elasticfleet/install_agent_grid.sls

@@ -14,12 +14,14 @@ run_installer:
     - name: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64
     - cwd: /opt/so
     - args: -token={{ GRIDNODETOKENGENERAL }}
+    - retry: True
 {% else %} 
 run_installer:
   cmd.script:
     - name: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64
     - cwd: /opt/so
     - args: -token={{ GRIDNODETOKENHEAVY }}
+    - retry: True
 {% endif %}  
 
 {% endif %}

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update

@@ -0,0 +1,53 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
+# this file except in compliance with the Elastic License 2.0.
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+
+. /usr/sbin/so-common
+
+# Only run on Managers
+if ! is_manager_node; then
+    printf "Not a Manager Node... Exiting"
+    exit 0
+fi
+
+function update_es_urls() {
+	# Generate updated JSON payload
+    JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"so-manager_elasticsearch","type":"elasticsearch","hosts": $UPDATEDLIST,"is_default":false,"is_default_monitoring":false,"config_yaml":""}')
+
+    # Update Fleet Elasticsearch URLs
+    curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+}
+
+# Get current list of Fleet Elasticsearch URLs
+RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_elasticsearch')
+
+# Check to make sure that the server responded with good data - else, bail from script
+CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON")
+if [ "$CHECKSUM" != "so-manager_elasticsearch" ]; then
+ printf "Failed to query for current Fleet Server Elasticsearch URLs..."
+ exit 1
+fi
+
+# Get the current list of Fleet Server Elasticsearch & hash them
+CURRENT_LIST=$(jq -c -r '.item.hosts' <<<  "$RAW_JSON")
+CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}')
+
+# Create array & add initial elements
+NEW_LIST=("https://{{ GLOBALS.hostname }}:9200")
+
+
+# Sort & hash the new list of Fleet Elasticsearch URLs
+NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}")
+NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
+
+# Compare the current & new list of URLs - if different, update the Fleet Elasticsearch URLs
+if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then
+    printf "\nHashes match - no update needed.\n"
+    printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n"
+    exit 0
+else
+    printf "\nHashes don't match - update needed.\n"
+    printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n"
+    update_es_urls
+fi

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update

@@ -4,6 +4,14 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% set CUSTOMFQDN = salt['pillar.get']('elasticfleet:config:server:custom_fqdn') %}
 
+. /usr/sbin/so-common
+
+# Only run on Managers
+if ! is_manager_node; then
+    printf "Not a Manager Node... Exiting"
+    exit 0
+fi
+
 function update_logstash_outputs() {
 	# Generate updated JSON payload
     JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":""}')
@@ -27,10 +35,10 @@ CURRENT_LIST=$(jq -c -r '.item.hosts' <<<  "$RAW_JSON")
 CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}')
 
 # Create array & add initial elements
-if [ "{{ GLOBALS.manager_ip }}" = "{{ GLOBALS.url_base }}" ]; then
+if [ "{{ GLOBALS.hostname }}" = "{{ GLOBALS.url_base }}" ]; then
     NEW_LIST=("{{ GLOBALS.url_base }}:5055")
 else
-    NEW_LIST=("{{ GLOBALS.url_base }}:5055" "{{ GLOBALS.manager_ip }}:5055")
+    NEW_LIST=("{{ GLOBALS.url_base }}:5055" "{{ GLOBALS.hostname }}:5055")
 fi
 
 {% if CUSTOMFQDN != "" %}

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup

@@ -35,12 +35,12 @@ curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fl
 printf "\n\n"
 {%- endif %}
 
-# Add Manager IP & URL Base to Fleet Host URLs
+# Add Manager Hostname & URL Base to Fleet Host URLs
 printf "\nAdd SO-Manager Fleet URL\n"
-if [ "{{ GLOBALS.manager_ip }}" = "{{ GLOBALS.url_base }}" ]; then
+if [ "{{ GLOBALS.hostname }}" = "{{ GLOBALS.url_base }}" ]; then
     JSON_STRING=$( jq -n '{"id":"grid-default","name":"grid-default","is_default":true,"host_urls":["https://{{ GLOBALS.url_base }}:8220"]}')
 else
-    JSON_STRING=$( jq -n '{"id":"grid-default","name":"grid-default","is_default":true,"host_urls":["https://{{ GLOBALS.url_base }}:8220", "https://{{ GLOBALS.manager_ip }}:8220"]}')
+    JSON_STRING=$( jq -n '{"id":"grid-default","name":"grid-default","is_default":true,"host_urls":["https://{{ GLOBALS.url_base }}:8220", "https://{{ GLOBALS.hostname }}:8220"]}')
 fi
 
 ## This array replaces whatever URLs are currently configured

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update

@@ -0,0 +1,74 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
+# this file except in compliance with the Elastic License 2.0.
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+{% set CUSTOMFQDN = salt['pillar.get']('elasticfleet:config:server:custom_fqdn') %}
+
+. /usr/sbin/so-common
+
+# Only run on Managers
+if ! is_manager_node; then
+    printf "Not a Manager Node... Exiting"
+    exit 0
+fi
+
+function update_fleet_urls() {
+	# Generate updated JSON payload
+    JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"grid-default","is_default":true,"host_urls": $UPDATEDLIST}')
+
+    # Update Fleet Server URLs
+    curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/fleet_server_hosts/grid-default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+}
+
+# Get current list of Fleet Server URLs
+RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts/grid-default')
+
+# Check to make sure that the server responded with good data - else, bail from script
+CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON")
+if [ "$CHECKSUM" != "grid-default" ]; then
+ printf "Failed to query for current Fleet Server URLs..."
+ exit 1
+fi
+
+# Get the current list of Fleet Server URLs & hash them
+CURRENT_LIST=$(jq -c -r '.item.host_urls' <<<  "$RAW_JSON")
+CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}')
+
+# Create array & add initial elements
+if [ "{{ GLOBALS.hostname }}" = "{{ GLOBALS.url_base }}" ]; then
+    NEW_LIST=("https://{{ GLOBALS.url_base }}:8220")
+else
+    NEW_LIST=("https://{{ GLOBALS.url_base }}:8220" "https://{{ GLOBALS.hostname }}:8220")
+fi
+
+{% if CUSTOMFQDN != "" %}
+# Add Custom Hostname to list
+NEW_LIST+=("https://{{ CUSTOMFQDN }}:8220")
+{% endif %}
+
+# Query for the current Grid Nodes that are running Logstash (which includes Fleet Nodes)
+LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local')
+
+# Query for Fleet Nodes & add them to the list (Hostname)
+if grep -q "fleet" <<< $LOGSTASHNODES; then
+   readarray -t FLEETNODES < <(jq -r ' .fleet | keys_unsorted[]'  <<< $LOGSTASHNODES)
+   for NODE in "${FLEETNODES[@]}"
+   do
+    NEW_LIST+=("https://$NODE:8220")
+   done
+fi
+
+# Sort & hash the new list of Fleet Server URLs
+NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}")
+NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
+
+# Compare the current & new list of URLs - if different, update the Fleet Server URLs
+if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then
+    printf "\nHashes match - no update needed.\n"
+    printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n"
+    exit 0
+else
+    printf "\nHashes don't match - update needed.\n"
+    printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n"
+    update_fleet_urls
+fi

salt/logstash/defaults.yaml

@@ -21,16 +21,16 @@ logstash:
         - fleet
   defined_pipelines:
     fleet:
-      - so/0012_input_elastic_agent.conf
+      - so/0012_input_elastic_agent.conf.jinja
       - so/9806_output_lumberjack_fleet.conf.jinja
     manager:
       - so/0011_input_endgame.conf
-      - so/0012_input_elastic_agent.conf 
+      - so/0012_input_elastic_agent.conf.jinja
       - so/0013_input_lumberjack_fleet.conf
       - so/9999_output_redis.conf.jinja        
     receiver:
       - so/0011_input_endgame.conf
-      - so/0012_input_elastic_agent.conf
+      - so/0012_input_elastic_agent.conf.jinja
       - so/0013_input_lumberjack_fleet.conf
       - so/9999_output_redis.conf.jinja
     search:

salt/logstash/pipelines/config/so/0012_input_elastic_agent.conf.jinja

@@ -1,7 +1,7 @@
 input {
   elastic_agent {
     port => 5055
-    tags => [ "elastic-agent" ]
+    tags => [ "elastic-agent", "input-{{ GLOBALS.hostname }}" ]
     ssl => true
     ssl_certificate_authorities => ["/usr/share/filebeat/ca.crt"]
     ssl_certificate => "/usr/share/logstash/elasticfleet-logstash.crt"