diff --git a/files/salt/master/master b/files/salt/master/master index 6cf3b94ea..2333a50aa 100644 --- a/files/salt/master/master +++ b/files/salt/master/master @@ -41,9 +41,9 @@ file_roots: base: - /opt/so/saltstack/local/salt - /opt/so/saltstack/default/salt + - /nsm/elastic-fleet/artifacts - /opt/so/rules/nids - # The master_roots setting configures a master-only copy of the file_roots dictionary, # used by the state compiler. # master_roots: /opt/so/saltstack/salt-master diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index d27f51ede..3ead8b26e 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -180,6 +180,7 @@ 'telegraf', 'firewall', 'logstash', + 'nginx', 'healthcheck', 'schedule', 'elasticfleet', diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index 9a27843ae..4bc212fbe 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -84,6 +84,13 @@ docker: custom_bind_mounts: [] extra_hosts: [] extra_env: [] + 'so-nginx-fleet-node': + final_octet: 31 + port_bindings: + - 8443:8443 + custom_bind_mounts: [] + extra_hosts: [] + extra_env: [] 'so-playbook': final_octet: 32 port_bindings: diff --git a/salt/docker/soc_docker.yaml b/salt/docker/soc_docker.yaml index 850324a9e..6e0efeb20 100644 --- a/salt/docker/soc_docker.yaml +++ b/salt/docker/soc_docker.yaml @@ -48,6 +48,7 @@ docker: so-logstash: *dockerOptions so-mysql: *dockerOptions so-nginx: *dockerOptions + so-nginx-fleet-node: *dockerOptions so-playbook: *dockerOptions so-redis: *dockerOptions so-sensoroni: *dockerOptions diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index 31c4e3469..4fc738171 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -38,12 +38,26 @@ so-elastic-fleet-auto-configure-server-urls: - retry: True {% endif %} -# Automatically update Fleet Server Elasticsearch URLs +# Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs {% if grains.role not in ['so-fleet'] %} so-elastic-fleet-auto-configure-elasticsearch-urls: cmd.run: - name: /usr/sbin/so-elastic-fleet-es-url-update - retry: True + +so-elastic-fleet-auto-configure-artifact-urls: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-artifacts-url-update + - retry: True + +{% endif %} + +# Sync Elastic Agent artifacts to Fleet Node +{% if grains.role in ['so-fleet'] %} +elasticagent_syncartifacts: + file.recurse: + - name: /nsm/elastic-fleet/artifacts/beats + - source: salt://beats {% endif %} {% if SERVICETOKEN != '' %} diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update new file mode 100644 index 000000000..721525668 --- /dev/null +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update @@ -0,0 +1,90 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use +# this file except in compliance with the Elastic License 2.0. +{% from 'vars/globals.map.jinja' import GLOBALS %} + +. /usr/sbin/so-common + +# Only run on Managers +if ! is_manager_node; then + printf "Not a Manager Node... Exiting" + exit 0 +fi + +# Function to check if an array contains a value +array_contains () { + local array="$1[@]" + local seeking=$2 + local in=1 + for element in "${!array}"; do + if [[ $element == "$seeking" ]]; then + in=0 + break + fi + done + return $in +} + +# Query for the current Grid Nodes that are running Logstash (which includes Fleet Nodes) +LOGSTASHNODES='{{ salt['pillar.get']('logstash:nodes', {}) | tojson }}' + +# Initialize an array for new hosts from Fleet Nodes +declare -a NEW_LIST=() + +# Query for Fleet Nodes & add them to the list (Hostname) +if grep -q "fleet" <<< "$LOGSTASHNODES"; then + readarray -t FLEETNODES < <(jq -r '.fleet | keys_unsorted[]' <<< "$LOGSTASHNODES") + for NODE in "${FLEETNODES[@]}"; do + URL="http://$NODE:8443/artifacts/" + NAME="FleetServer_$NODE" + NEW_LIST+=("$URL=$NAME") + done +fi + +# Create an array for expected hosts and their names +declare -A expected_urls=( + ["http://{{ GLOBALS.url_base }}:8443/artifacts/"]="FleetServer_{{ GLOBALS.hostname }}" + ["https://artifacts.elastic.co/downloads/"]="Elastic Artifacts" +) + +# Merge NEW_LIST into expected_urls +for entry in "${NEW_LIST[@]}"; do + # Extract URL and Name from each entry + IFS='=' read -r URL NAME <<< "$entry" + # Add to expected_urls, automatically handling URL as key and NAME as value + expected_urls["$URL"]="$NAME" +done + +# Fetch the current hosts from the API +current_urls=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/agent_download_sources' | jq -r .items[].host) + +# Convert current hosts to an array +IFS=$'\n' read -rd '' -a current_urls_array <<<"$current_urls" + +# Flag to track if any host was added +any_url_added=0 + +# Check each expected host +for host in "${!expected_urls[@]}"; do + array_contains current_urls_array "$host" || { + echo "$host (${expected_urls[$host]}) is missing. Adding it..." + + # Prepare the JSON payload + JSON_STRING=$( jq -n \ + --arg NAME "${expected_urls[$host]}" \ + --arg URL "$host" \ + '{"name":$NAME,"host":$URL}' ) + + # Create the missing host + curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_download_sources" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" + + # Flag that an artifact URL was added + any_url_added=1 + } + +done + + +if [[ $any_url_added -eq 0 ]]; then + echo "All expected artifact URLs are present. No updates needed." +fi diff --git a/salt/firewall/containers.map.jinja b/salt/firewall/containers.map.jinja index 0ba2389e9..b3ead0f4c 100644 --- a/salt/firewall/containers.map.jinja +++ b/salt/firewall/containers.map.jinja @@ -95,6 +95,7 @@ {% set NODE_CONTAINERS = [ 'so-elastic-fleet', 'so-logstash', + 'so-nginx-fleet-node' ] %} {% elif GLOBALS.role == 'so-sensor' %} diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index d93218db4..600cb5d4e 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -610,6 +610,9 @@ up_to_2.4.50() { mkdir /opt/so/rules/nids/suri chown socore:socore /opt/so/rules/nids/suri mv -v /opt/so/rules/nids/*.rules /opt/so/rules/nids/suri/. + + echo "Adding /nsm/elastic-fleet/artifacts to file_roots in /etc/salt/master using so-yaml" + so-yaml.py append /etc/salt/master file_roots.base /nsm/elastic-fleet/artifacts INSTALLEDVERSION=2.4.50 } diff --git a/salt/nginx/enabled.sls b/salt/nginx/enabled.sls index dda475655..273fb65be 100644 --- a/salt/nginx/enabled.sls +++ b/salt/nginx/enabled.sls @@ -14,6 +14,9 @@ include: - nginx.config - nginx.sostatus + +{% if grains.role not in ['so-fleet'] %} + {# if the user has selected to replace the crt and key in the ui #} {% if NGINXMERGED.ssl.replace_cert %} @@ -88,6 +91,15 @@ make-rule-dir-nginx: - recurse: - user - group + +{% endif %} + +{# if this is an so-fleet node then we want to use the port bindings, custom bind mounts defined for fleet #} +{% if GLOBALS.role == 'so-fleet' %} +{% set container_config = 'so-nginx-fleet-node' %} +{% else %} +{% set container_config = 'so-nginx' %} +{% endif %} so-nginx: docker_container.running: @@ -95,11 +107,11 @@ so-nginx: - hostname: so-nginx - networks: - sobridge: - - ipv4_address: {{ DOCKER.containers['so-nginx'].ip }} + - ipv4_address: {{ DOCKER.containers[container_config].ip }} - extra_hosts: - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }} - {% if DOCKER.containers['so-nginx'].extra_hosts %} - {% for XTRAHOST in DOCKER.containers['so-nginx'].extra_hosts %} + {% if DOCKER.containers[container_config].extra_hosts %} + {% for XTRAHOST in DOCKER.containers[container_config].extra_hosts %} - {{ XTRAHOST }} {% endfor %} {% endif %} @@ -119,20 +131,20 @@ so-nginx: - /nsm/repo:/opt/socore/html/repo:ro - /nsm/rules:/nsm/rules:ro {% endif %} - {% if DOCKER.containers['so-nginx'].custom_bind_mounts %} - {% for BIND in DOCKER.containers['so-nginx'].custom_bind_mounts %} + {% if DOCKER.containers[container_config].custom_bind_mounts %} + {% for BIND in DOCKER.containers[container_config].custom_bind_mounts %} - {{ BIND }} {% endfor %} {% endif %} - {% if DOCKER.containers['so-nginx'].extra_env %} + {% if DOCKER.containers[container_config].extra_env %} - environment: - {% for XTRAENV in DOCKER.containers['so-nginx'].extra_env %} + {% for XTRAENV in DOCKER.containers[container_config].extra_env %} - {{ XTRAENV }} {% endfor %} {% endif %} - cap_add: NET_BIND_SERVICE - port_bindings: - {% for BINDING in DOCKER.containers['so-nginx'].port_bindings %} + {% for BINDING in DOCKER.containers[container_config].port_bindings %} - {{ BINDING }} {% endfor %} - watch: diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index d5981be77..236f8da7f 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -39,6 +39,26 @@ http { include /etc/nginx/conf.d/*.conf; + {%- if role in ['fleet'] %} + + server { + listen 8443; + server_name {{ GLOBALS.hostname }}; + root /opt/socore/html; + location /artifacts/ { + try_files $uri =206; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; + } + } + + {%- endif %} + {%- if role in ['eval', 'managersearch', 'manager', 'standalone', 'import'] %} server { diff --git a/salt/top.sls b/salt/top.sls index f8979956e..16b355476 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -264,6 +264,7 @@ base: - telegraf - firewall - logstash + - nginx - elasticfleet - elasticfleet.install_agent_grid - schedule