From 7561ec05127c796e4d48077a17ae4cd4865c9762 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 29 Jun 2023 08:52:51 -0400 Subject: [PATCH] Automatically manage Fleet Logstash Config --- salt/elasticfleet/defaults.yaml | 3 +- salt/elasticfleet/enabled.sls | 11 +++ salt/elasticfleet/soc_elasticfleet.yaml | 15 ++-- .../so-elastic-fleet-outputs-update | 75 +++++++++++++++++++ salt/manager/tools/sbin/so-minion | 2 +- 5 files changed, 99 insertions(+), 7 deletions(-) create mode 100644 salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 83261dc17..93b5eba9a 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -2,10 +2,11 @@ elasticfleet: enabled: False config: server: + custom_fqdn: '' + enable_auto_configuration: True endpoints_enrollment: '' es_token: '' grid_enrollment: '' - url: '' logging: zeek: excluded: diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index f388cb1c7..17dc8afa0 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -9,11 +9,22 @@ {% from 'docker/docker.map.jinja' import DOCKER %} {# This value is generated during node install and stored in minion pillar #} {% set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %} +{% set ENABLEAUTOCONFIGURATION = salt['pillar.get']('elasticfleet:config:server:enable_auto_configuration','') %} include: - elasticfleet.config - elasticfleet.sostatus +{% if ENABLEAUTOCONFIGURATION %} +so-elastic-fleet-auto-configure-logstash-outputs: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-outputs-update + +#so-elastic-fleet-auto-configure-server-urls: +# cmd.run: +# - name: /usr/sbin/so-elastic-fleet-urls-update +{% endif %} + {% if SERVICETOKEN != '' %} so-elastic-fleet: docker_container.running: diff --git a/salt/elasticfleet/soc_elasticfleet.yaml b/salt/elasticfleet/soc_elasticfleet.yaml index 80b3a22b5..9b918f0ac 100644 --- a/salt/elasticfleet/soc_elasticfleet.yaml +++ b/salt/elasticfleet/soc_elasticfleet.yaml @@ -11,6 +11,16 @@ elasticfleet: helpLink: zeek.html config: server: + custom_fqdn: + description: Custom FQDN for Agents to connect to. + global: True + helpLink: elastic-fleet.html + advanced: True + enable_auto_configuration: + description: Enable auto-configuration of Logstash Outputs & Fleet Host URLs. + global: True + helpLink: elastic-fleet.html + advanced: True endpoints_enrollment: description: Endpoint enrollment key. global: True @@ -29,8 +39,3 @@ elasticfleet: helpLink: elastic-fleet.html sensitive: True advanced: True - url: - description: Agent connection URL. - global: True - helpLink: elastic-fleet.html - advanced: True diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update new file mode 100644 index 000000000..b4df64d68 --- /dev/null +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update @@ -0,0 +1,75 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use +# this file except in compliance with the Elastic License 2.0. +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% set CUSTOMFQDN = salt['pillar.get']('elasticfleet:config:server:custom_fqdn') %} + +function update_logstash_outputs() { + # Generate updated JSON payload + JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":""}') + + # Update Logstash Outputs + curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_logstash" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" | jq +} + +# Get current list of Logstash Outputs +RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_logstash') + +# Check to make sure that the server responded with good data - else, bail from script +CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON") +if [ "$CHECKSUM" != "so-manager_logstash" ]; then + printf "Failed to query for current Logstash Outputs..." + exit 1 +fi + +# Get the current list of Logstash outputs & hash them +CURRENT_LIST=$(jq -c -r '.item.hosts' <<< "$RAW_JSON") +CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}') + +# Create array & add initial elements +if [ "{{ GLOBALS.manager_ip }}" = "{{ GLOBALS.url_base }}" ]; then + NEW_LIST=("{{ GLOBALS.url_base }}:5055") +else + NEW_LIST=("{{ GLOBALS.url_base }}:5055" "{{ GLOBALS.manager_ip }}:5055") +fi + +{% if CUSTOMFQDN != "" %} +# Add Custom Hostname to list +NEW_LIST+=("{{ CUSTOMFQDN }}:5055") +{% endif %} + +# Query for the current Grid Nodes that are running Logstash +LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local') + +# Query for Receiver Nodes & add them to the list +if grep -q "receiver" <<< $LOGSTASHNODES; then + readarray -t RECEIVERNODES < <(jq -r ' .receiver | keys_unsorted[]' <<< $LOGSTASHNODES) + for NODE in "${RECEIVERNODES[@]}" + do + NEW_LIST+=("$NODE:5055") + done +fi + +# Query for Fleet Nodes & add them to the list +if grep -q "fleet" <<< $LOGSTASHNODES; then + readarray -t FLEETNODES < <(jq -r ' .fleet | keys_unsorted[]' <<< $LOGSTASHNODES) + for NODE in "${FLEETNODES[@]}" + do + NEW_LIST+=("$NODE:5055") + done +fi + +# Sort & hash the new list of Logstash Outputs +NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}") +NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}') + +# Compare the current & new list of outputs - if different, update the Logstash outputs +if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then + printf "\nHashes match - no update needed.\n" + printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" + exit 0 +else + printf "\nHashes don't match - update needed.\n" + printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" + update_logstash_outputs +fi diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index ad2188644..edc0b1404 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -515,7 +515,7 @@ function createFLEET() { add_logstash_to_minion create_fleet_policy update_fleet_host_urls - update_logstash_outputs + #update_logstash_outputs add_telegraf_to_minion add_nginx_to_minion }