Merge pull request #12268 from Security-Onion-Solutions/feature/fleet-artifacts

Feature/fleet artifacts

files/salt/master/master

@@ -41,9 +41,9 @@ file_roots:
   base:
     - /opt/so/saltstack/local/salt
     - /opt/so/saltstack/default/salt
+    - /nsm/elastic-fleet/artifacts
     - /opt/so/rules/nids
 
-
 # The master_roots setting configures a master-only copy of the file_roots dictionary,
 # used by the state compiler.
 # master_roots: /opt/so/saltstack/salt-master

salt/allowed_states.map.jinja

@@ -180,6 +180,7 @@
           'telegraf',
           'firewall',
           'logstash',
+          'nginx',
           'healthcheck',
           'schedule',
           'elasticfleet',

salt/docker/defaults.yaml

@@ -84,6 +84,13 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+    'so-nginx-fleet-node':
+      final_octet: 31
+      port_bindings:
+        - 8443:8443
+      custom_bind_mounts: []
+      extra_hosts: []
+      extra_env: []
     'so-playbook':
       final_octet: 32
       port_bindings:

salt/docker/soc_docker.yaml

@@ -48,6 +48,7 @@ docker:
     so-logstash: *dockerOptions
     so-mysql: *dockerOptions
     so-nginx: *dockerOptions
+    so-nginx-fleet-node: *dockerOptions
     so-playbook: *dockerOptions
     so-redis: *dockerOptions
     so-sensoroni: *dockerOptions

salt/elasticfleet/enabled.sls

@@ -38,12 +38,26 @@ so-elastic-fleet-auto-configure-server-urls:
     - retry: True
 {% endif %}
 
-# Automatically update Fleet Server Elasticsearch URLs
+# Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs
 {% if grains.role not in ['so-fleet'] %}
 so-elastic-fleet-auto-configure-elasticsearch-urls:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-es-url-update
     - retry: True
+
+so-elastic-fleet-auto-configure-artifact-urls:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-artifacts-url-update
+    - retry: True 
+
+{% endif %}
+
+# Sync Elastic Agent artifacts to Fleet Node
+{% if grains.role in ['so-fleet'] %}
+elasticagent_syncartifacts:
+  file.recurse:
+    - name: /nsm/elastic-fleet/artifacts/beats
+    - source: salt://beats
 {% endif %}
 
 {%   if SERVICETOKEN != '' %}

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update

@@ -0,0 +1,90 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
+# this file except in compliance with the Elastic License 2.0.
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+
+. /usr/sbin/so-common
+
+# Only run on Managers
+if ! is_manager_node; then
+    printf "Not a Manager Node... Exiting"
+    exit 0
+fi
+
+# Function to check if an array contains a value
+array_contains () { 
+    local array="$1[@]"
+    local seeking=$2
+    local in=1
+    for element in "${!array}"; do
+        if [[ $element == "$seeking" ]]; then
+            in=0
+            break
+        fi
+    done
+    return $in
+}
+
+# Query for the current Grid Nodes that are running Logstash (which includes Fleet Nodes)
+LOGSTASHNODES='{{ salt['pillar.get']('logstash:nodes', {}) | tojson }}'
+
+# Initialize an array for new hosts from Fleet Nodes
+declare -a NEW_LIST=()
+
+# Query for Fleet Nodes & add them to the list (Hostname)
+if grep -q "fleet" <<< "$LOGSTASHNODES"; then
+    readarray -t FLEETNODES < <(jq -r '.fleet | keys_unsorted[]' <<< "$LOGSTASHNODES")
+    for NODE in "${FLEETNODES[@]}"; do
+        URL="http://$NODE:8443/artifacts/"
+        NAME="FleetServer_$NODE"
+        NEW_LIST+=("$URL=$NAME")
+    done
+fi
+
+# Create an array for expected hosts and their names
+declare -A expected_urls=(
+    ["http://{{ GLOBALS.url_base }}:8443/artifacts/"]="FleetServer_{{ GLOBALS.hostname }}"
+    ["https://artifacts.elastic.co/downloads/"]="Elastic Artifacts"
+)
+
+# Merge NEW_LIST into expected_urls
+for entry in "${NEW_LIST[@]}"; do
+    # Extract URL and Name from each entry
+    IFS='=' read -r URL NAME <<< "$entry"
+    # Add to expected_urls, automatically handling URL as key and NAME as value
+    expected_urls["$URL"]="$NAME"
+done
+
+# Fetch the current hosts from the API
+current_urls=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/agent_download_sources' | jq -r .items[].host)
+
+# Convert current hosts to an array
+IFS=$'\n' read -rd '' -a current_urls_array <<<"$current_urls"
+
+# Flag to track if any host was added
+any_url_added=0
+
+# Check each expected host
+for host in "${!expected_urls[@]}"; do
+    array_contains current_urls_array "$host" || {
+        echo "$host (${expected_urls[$host]}) is missing. Adding it..."
+        
+        # Prepare the JSON payload
+        JSON_STRING=$( jq -n \
+                        --arg NAME "${expected_urls[$host]}" \
+                        --arg URL "$host" \
+                        '{"name":$NAME,"host":$URL}' )
+        
+        # Create the missing host
+        curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_download_sources" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+        
+        # Flag that an artifact URL was added
+        any_url_added=1
+    }
+
+done
+
+
+if [[ $any_url_added -eq 0 ]]; then
+    echo "All expected artifact URLs are present. No updates needed."
+fi

salt/firewall/containers.map.jinja

@@ -95,6 +95,7 @@
 {% set NODE_CONTAINERS = [
      'so-elastic-fleet',
      'so-logstash',
+     'so-nginx-fleet-node'
 ] %}
 
 {% elif GLOBALS.role == 'so-sensor' %}

salt/manager/tools/sbin/soup

@@ -610,6 +610,9 @@ up_to_2.4.50() {
   mkdir /opt/so/rules/nids/suri
   chown socore:socore /opt/so/rules/nids/suri
   mv -v /opt/so/rules/nids/*.rules /opt/so/rules/nids/suri/.
+  
+  echo "Adding /nsm/elastic-fleet/artifacts to file_roots in /etc/salt/master using so-yaml"
+  so-yaml.py append /etc/salt/master file_roots.base  /nsm/elastic-fleet/artifacts
 
   INSTALLEDVERSION=2.4.50
 }

salt/nginx/enabled.sls

@@ -14,6 +14,9 @@ include:
   - nginx.config
   - nginx.sostatus
 
+
+{%   if grains.role not in ['so-fleet'] %}
+
 {#   if the user has selected to replace the crt and key in the ui #}
 {%   if NGINXMERGED.ssl.replace_cert %}
 
@@ -88,6 +91,15 @@ make-rule-dir-nginx:
     - recurse:
       - user
       - group
+      
+{%   endif %}
+
+{# if this is an so-fleet node then we want to use the port bindings, custom bind mounts defined for fleet #}
+{% if GLOBALS.role == 'so-fleet' %}
+{%   set container_config = 'so-nginx-fleet-node' %}
+{% else %}
+{%   set container_config = 'so-nginx' %}
+{% endif %}
 
 so-nginx:
   docker_container.running:
@@ -95,11 +107,11 @@ so-nginx:
     - hostname: so-nginx
     - networks:
       - sobridge:
-        - ipv4_address: {{ DOCKER.containers['so-nginx'].ip }}
+        - ipv4_address: {{ DOCKER.containers[container_config].ip }}
     - extra_hosts:
       - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }}
-    {% if DOCKER.containers['so-nginx'].extra_hosts %}
+    {% if DOCKER.containers[container_config].extra_hosts %}
-      {% for XTRAHOST in DOCKER.containers['so-nginx'].extra_hosts %}
+      {% for XTRAHOST in DOCKER.containers[container_config].extra_hosts %}
       - {{ XTRAHOST }}
       {% endfor %}
     {% endif %}
@@ -119,20 +131,20 @@ so-nginx:
       - /nsm/repo:/opt/socore/html/repo:ro
       - /nsm/rules:/nsm/rules:ro
       {% endif %}
-      {% if DOCKER.containers['so-nginx'].custom_bind_mounts %}
+      {% if DOCKER.containers[container_config].custom_bind_mounts %}
-        {% for BIND in DOCKER.containers['so-nginx'].custom_bind_mounts %}
+        {% for BIND in DOCKER.containers[container_config].custom_bind_mounts %}
       - {{ BIND }}
         {% endfor %}
       {% endif %}
-    {% if DOCKER.containers['so-nginx'].extra_env %}
+    {% if DOCKER.containers[container_config].extra_env %}
     - environment:
-      {% for XTRAENV in DOCKER.containers['so-nginx'].extra_env %}
+      {% for XTRAENV in DOCKER.containers[container_config].extra_env %}
       - {{ XTRAENV }}
       {% endfor %}
     {% endif %}
     - cap_add: NET_BIND_SERVICE
     - port_bindings:
-      {% for BINDING in DOCKER.containers['so-nginx'].port_bindings %}
+      {% for BINDING in DOCKER.containers[container_config].port_bindings %}
       - {{ BINDING }}
       {% endfor %}
     - watch:

salt/nginx/etc/nginx.conf

@@ -39,6 +39,26 @@ http {
 
 	include /etc/nginx/conf.d/*.conf;
 
+	{%- if role in ['fleet'] %}
+
+	server {
+		listen 8443;
+		server_name {{ GLOBALS.hostname }};
+		root /opt/socore/html;
+		location /artifacts/ {
+			try_files $uri =206;
+			proxy_read_timeout    90;
+			proxy_connect_timeout 90;
+			proxy_set_header      Host $host;
+			proxy_set_header      X-Real-IP $remote_addr;
+			proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
+			proxy_set_header      Proxy "";
+			proxy_set_header      X-Forwarded-Proto $scheme;
+		}
+	}
+
+    {%- endif %}
+
 	{%- if role in ['eval', 'managersearch', 'manager', 'standalone', 'import'] %}
 
 	server {

salt/top.sls

@@ -264,6 +264,7 @@ base:
     - telegraf
     - firewall
     - logstash
+    - nginx
     - elasticfleet
     - elasticfleet.install_agent_grid
     - schedule