From afa98fa147f9b66a97692a35917f6dc766b3b93f Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Sun, 28 Jan 2024 14:20:52 -0500 Subject: [PATCH] update artifacts URL automatically --- salt/elasticfleet/enabled.sls | 7 +- .../so-elastic-fleet-artifacts-url-update | 102 ++++++++++++++++++ salt/firewall/containers.map.jinja | 1 + salt/nginx/enabled.sls | 5 + 4 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index bd8ab51c0..dca7f479f 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -33,12 +33,17 @@ so-elastic-fleet-auto-configure-server-urls: - retry: True {% endif %} -# Automatically update Fleet Server Elasticsearch URLs +# Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs {% if grains.role not in ['so-fleet'] %} so-elastic-fleet-auto-configure-elasticsearch-urls: cmd.run: - name: /usr/sbin/so-elastic-fleet-es-url-update - retry: True + +so-elastic-fleet-auto-configure-elasticsearch-urls: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-artifacts-url-update + - retry: True {% endif %} # Sync Elastic Agent artifacts to Fleet Node diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update new file mode 100644 index 000000000..685db392f --- /dev/null +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update @@ -0,0 +1,102 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use +# this file except in compliance with the Elastic License 2.0. +{% from 'vars/globals.map.jinja' import GLOBALS %} + +. /usr/sbin/so-common + +# Only run on Managers +if ! is_manager_node; then + printf "Not a Manager Node... Exiting" + exit 0 +fi + +########## +# Set Elastic Agent Artifact Registry URL + + + + +function update_es_urls() { + +# For each element in NEWLIST, create a new entry + +JSON_STRING=$( jq -n \ + --arg NAME "FleetServer_{{ GLOBALS.hostname }}" \ + --arg URL "http://{{ GLOBALS.url_base }}:8443/artifacts/" \ + '{"name":$NAME,"host":$URL,"is_default":true}' + ) + +curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_download_sources" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" + +######### + + # Generate updated JSON payload +{% if grains.role not in ['so-import', 'so-eval'] %} + JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"so-manager_elasticsearch","type":"elasticsearch","hosts": $UPDATEDLIST,"config_yaml":""}') +{%- else %} + JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"so-manager_elasticsearch","type":"elasticsearch","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":""}') +{%- endif %} + # Update Fleet Elasticsearch URLs + curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" +} + +#START HERE + +# Get current list of Artifact URLs +#RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_elasticsearch') +RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/agent_download_sources') + + +# Check to make sure that the server responded with good data - else, bail from script +CHECKSUM=$(jq -r '.page' <<< "$RAW_JSON") +if [ "$CHECKSUM" != "1" ]; then + printf "Failed to query for current Elastic Agent Artifact URLs..." + exit 1 +fi + +# Get the current list of Elastic Agent Artifact URLs & hash them +CURRENT_LIST=$(jq -c -r '.items[].host' <<< "$RAW_JSON") +CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}') + + +# Create array & add initial elements +if [ "{{ GLOBALS.hostname }}" = "{{ GLOBALS.url_base }}" ]; then + NEW_LIST=("http://{{ GLOBALS.url_base }}:8443/artifacts/") +else + NEW_LIST=("http://{{ GLOBALS.url_base }}:8443/artifacts/" "http://{{ GLOBALS.hostname }}:8443/artifacts/") +fi + +# Query for the current Grid Nodes that are running Logstash (which includes Fleet Nodes) +LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local') + +# Query for Fleet Nodes & add them to the list (Hostname) +if grep -q "fleet" <<< $LOGSTASHNODES; then + readarray -t FLEETNODES < <(jq -r ' .fleet | keys_unsorted[]' <<< $LOGSTASHNODES) + for NODE in "${FLEETNODES[@]}" + do + NEW_LIST+=("http://$NODE:8443/artifacts/") + done +fi + +# Sort & hash the new list of Fleet Elasticsearch URLs +NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}") +NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}') + +# Compare the current & new list of URLs - if different, update the Fleet Elasticsearch URLs +if [ "$1" = "--force" ]; then + printf "\nUpdating List, since --force was specified.\n" + printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" + update_es_urls + exit 0 +fi + +if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then + printf "\nHashes match - no update needed.\n" + printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" + exit 0 +else + printf "\nHashes don't match - update needed.\n" + printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" + #update_es_urls +fi diff --git a/salt/firewall/containers.map.jinja b/salt/firewall/containers.map.jinja index 0ba2389e9..99a3bd5d0 100644 --- a/salt/firewall/containers.map.jinja +++ b/salt/firewall/containers.map.jinja @@ -95,6 +95,7 @@ {% set NODE_CONTAINERS = [ 'so-elastic-fleet', 'so-logstash', + 'so-nginx' ] %} {% elif GLOBALS.role == 'so-sensor' %} diff --git a/salt/nginx/enabled.sls b/salt/nginx/enabled.sls index dda475655..eca9c237a 100644 --- a/salt/nginx/enabled.sls +++ b/salt/nginx/enabled.sls @@ -14,6 +14,9 @@ include: - nginx.config - nginx.sostatus + +{% if grains.role not in ['so-fleet'] %} + {# if the user has selected to replace the crt and key in the ui #} {% if NGINXMERGED.ssl.replace_cert %} @@ -88,6 +91,8 @@ make-rule-dir-nginx: - recurse: - user - group + +{% endif %} so-nginx: docker_container.running: