Merge remote-tracking branch 'origin/2.4/dev' into 2.4/FleetEnhancments

salt/common/packages.sls

@@ -17,6 +17,7 @@ commonpkgs:
       - netcat-openbsd
       - sqlite3
       - libssl-dev
+      - procps
       - python3-dateutil
       - python3-docker
       - python3-packaging
@@ -70,6 +71,7 @@ commonpkgs:
       - net-tools
       - nmap-ncat
       - openssl
+      - procps-ng
       - python3-dnf-plugin-versionlock
       - python3-docker
       - python3-m2crypto

salt/common/tools/sbin/so-status

@@ -103,7 +103,7 @@ def output(options, console, code, data):
 def check_container_status(options, console):
   code = 0
   cli = "docker"
-  proc = subprocess.run([cli, 'ps', '--format', '{{json .}}'], stdout=subprocess.PIPE, encoding="utf-8")
+  proc = subprocess.run([cli, 'ps', '--format', 'json'], stdout=subprocess.PIPE, encoding="utf-8")
   if proc.returncode != 0:
     fail("Container system error; unable to obtain container process statuses")
 

salt/desktop/packages.sls

@@ -181,6 +181,7 @@ desktop_packages:
       - gstreamer1-plugins-good-gtk
       - gstreamer1-plugins-ugly-free
       - gtk-update-icon-cache
+      - gtk2
       - gtk3
       - gtk4
       - gtkmm30
@@ -295,6 +296,7 @@ desktop_packages:
       - mesa-vulkan-drivers
       - microcode_ctl
       - mobile-broadband-provider-info
+      - mono-devel
       - mpfr
       - mpg123-libs
       - mtdev

salt/elasticagent/config.sls

@@ -28,6 +28,13 @@ elasticagentconfdir:
     - group: 939
     - makedirs: True
 
+elasticagentlogdir:
+   file.directory:
+     - name: /opt/so/log/elasticagent
+     - user: 949
+     - group: 939
+     - makedirs: True
+
 elasticagent_sbin_jinja:
   file.recurse:
     - name: /usr/sbin

salt/elasticagent/enabled.sls

@@ -33,6 +33,7 @@ so-elastic-agent:
         {% endif %}
     - binds:
       - /opt/so/conf/elastic-agent/elastic-agent.yml:/usr/share/elastic-agent/elastic-agent.yml:ro
+      - /opt/so/log/elasticagent:/usr/share/elastic-agent/logs
       - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro 
       - /nsm:/nsm:ro
       - /opt/so/log:/opt/so/log:ro
@@ -43,6 +44,7 @@ so-elastic-agent:
       {% endif %}
     - environment:
       - FLEET_CA=/etc/pki/tls/certs/intca.crt
+      - LOGS_PATH=logs
       {% if DOCKER.containers['so-elastic-agent'].extra_env %}
         {% for XTRAENV in DOCKER.containers['so-elastic-agent'].extra_env %}
       - {{ XTRAENV }}

salt/elasticfleet/files/integrations/endpoints-initial/elastic-defend-endpoints.json

@@ -5,7 +5,7 @@
 	"package": {
 		"name": "endpoint",
 		"title": "Elastic Defend",
-		"version": ""
+		"version": "8.8.0"
 	},
 	"enabled": true,
 	"policy_id": "endpoints-initial",

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -15,10 +15,8 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
     printf "\n\nInitial Endpoints Policy - Loading $INTEGRATION\n"
     elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"
     if [ -n "$INTEGRATION_ID" ]; then
-      if [ "$NAME" != "elastic-defend-endpoints" ]; then
+      printf "\n\nIntegration $NAME exists - Updating integration\n"
-        printf "\n\nIntegration $NAME exists - Updating integration\n"
+      elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"
-        elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"
-      fi
     else
       printf "\n\nIntegration does not exist - Creating integration\n"
       elastic_fleet_integration_create "@$INTEGRATION"

salt/elasticfleet/tools/sbin/so-elastic-fleet-package-list

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-elastic-fleet-common
+
+# Let's snag a cookie from Kibana
+SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
+
+# List configured package policies
+curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages" -H 'kbn-xsrf: true' | jq
+
+echo

salt/elasticsearch/files/ingest/filterlog

@@ -49,11 +49,10 @@
                 "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}]
         }
      },
-     { "set":         { "field": "_index",        "value": "so-firewall", "override": true           } },
      { "set":         { "if": "ctx.network?.transport_id == '0'", "field": "network.transport",        "value": "icmp", "override": true           } },
      { "community_id": {} },
-     { "set":         { "field": "module",	"value": "pfsense",	"override": true	} },
+     { "set":         { "field": "event.module",	"value": "pfsense",	"override": true	} },
-     { "set":         { "field": "dataset",	"value": "firewall",	"override": true	} },
+     { "set":         { "field": "event.dataset",	"value": "firewall",	"override": true	} },
      { "set":         { "field": "category",	"value": "network",	"override": true	} },
      { "remove":      { "field": ["real_message", "ip_sub_msg", "firewall.sub_message"],     "ignore_failure": true  } }
   ]

salt/firewall/defaults.yaml

@@ -198,9 +198,6 @@ firewall:
               portgroups:
                 - redis
                 - elasticsearch_node
-            self:
-              portgroups:
-                - syslog
             beats_endpoint:
               portgroups:
                 - beats_5044
@@ -218,9 +215,6 @@ firewall:
             strelka_frontend:
               portgroups:
                 - strelka_frontend
-            syslog:
-              portgroups:
-                - syslog
             analyst:
               portgroups:
                 - nginx
@@ -255,6 +249,12 @@ firewall:
             localhost:
               portgroups:
                 - all
+            self:
+              portgroups:
+                - syslog
+            syslog:
+              portgroups:
+                - syslog
             customhostgroup0:
               portgroups: []
             customhostgroup1:
@@ -425,12 +425,6 @@ firewall:
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
-            self:
-              portgroups:
-                - syslog
-            syslog:
-              portgroups:
-                - syslog
             beats_endpoint:
               portgroups:
                 - beats_5044
@@ -497,6 +491,12 @@ firewall:
             receiver:
               portgroups:
                 - salt_manager
+            self:
+              portgroups:
+                - syslog
+            syslog:
+              portgroups:
+                - syslog
             customhostgroup0:
               portgroups: []
             customhostgroup1:
@@ -588,9 +588,6 @@ firewall:
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
-            self:
-              portgroups:
-                - syslog
             beats_endpoint:
               portgroups:
                 - beats_5044
@@ -608,9 +605,6 @@ firewall:
             endgame:
               portgroups:
                 - endgame
-            syslog:
-              portgroups:
-                - syslog
             analyst:
               portgroups:
                 - nginx
@@ -660,6 +654,12 @@ firewall:
             receiver:
               portgroups:
                 - salt_manager
+            self:
+              portgroups:
+                - syslog
+            syslog:
+              portgroups:
+                - syslog
             customhostgroup0:
               portgroups: []
             customhostgroup1:
@@ -760,9 +760,6 @@ firewall:
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
-            self:
-              portgroups:
-                - syslog
             beats_endpoint:
               portgroups:
                 - beats_5044
@@ -783,9 +780,6 @@ firewall:
             strelka_frontend:
               portgroups:
                 - strelka_frontend
-            syslog:
-              portgroups:
-                - syslog
             analyst:
               portgroups:
                 - nginx
@@ -838,6 +832,12 @@ firewall:
             receiver:
               portgroups:
                 - salt_manager
+            self:
+              portgroups:
+                - syslog
+            syslog:
+              portgroups:
+                - syslog
             customhostgroup0:
               portgroups: []
             customhostgroup1:
@@ -884,9 +884,6 @@ firewall:
             searchnode:
               portgroups:
                 - elasticsearch_node
-            self:
-              portgroups:
-                - syslog
             customhostgroup0:
               portgroups: []
             customhostgroup1:
@@ -918,6 +915,12 @@ firewall:
             localhost:
               portgroups:
                 - all
+            self:
+              portgroups:
+                - syslog
+            syslog:
+              portgroups:
+                - syslog
             customhostgroup0:
               portgroups: []
             customhostgroup1:
@@ -942,9 +945,6 @@ firewall:
       chain:
         DOCKER-USER:
           hostgroups:
-            self:
-              portgroups:
-                - syslog
             strelka_frontend:
               portgroups:
                 - strelka_frontend
@@ -979,6 +979,12 @@ firewall:
             localhost:
               portgroups:
                 - all
+            self:
+              portgroups:
+                - syslog
+            syslog:
+              portgroups:
+                - syslog
             customhostgroup0:
               portgroups: []
             customhostgroup1:
@@ -1030,6 +1036,9 @@ firewall:
             strelka_frontend:
               portgroups:
                 - strelka_frontend
+            syslog:
+              portgroups:
+                - syslog
             customhostgroup0:
               portgroups: []
             customhostgroup1:
@@ -1189,11 +1198,7 @@ firewall:
             self:
               portgroups:
                 - redis
-                - syslog
                 - beats_5644
-            syslog:
-              portgroups:
-                - syslog
             beats_endpoint:
               portgroups:
                 - beats_5044
@@ -1234,6 +1239,12 @@ firewall:
             localhost:
               portgroups:
                 - all
+            self:
+              portgroups:
+                - syslog
+            syslog:
+              portgroups:
+                - syslog
             customhostgroup0:
               portgroups: []
             customhostgroup1:

salt/firewall/soc_firewall.yaml

@@ -191,6 +191,7 @@ firewall:
                 description: Portgroups to add access to the docker containers for this role.
                 advanced: True
                 multiline: True
+                forcedType: "[]string"
                 helpLink: firewall.html
             sensor:
               portgroups: *portgroupsdocker
@@ -241,6 +242,7 @@ firewall:
                 description: Portgroups to add access to the host.
                 advanced: True
                 multiline: True
+                forcedType: "[]string"
                 helpLink: firewall.html
             dockernet: 
               portgroups: *portgroupshost

salt/logrotate/defaults.yaml

@@ -90,6 +90,26 @@ logrotate:
       - extension .log
       - dateext
       - dateyesterday
+    /opt/so/log/elasticagent/*_x_log:
+      - daily
+      - rotate 14
+      - missingok
+      - copytruncate
+      - compress
+      - create
+      - extension .log
+      - dateext
+      - dateyesterday
+    /opt/so/log/elasticagent/*_x_ndjson:
+      - daily
+      - rotate 14
+      - missingok
+      - copytruncate
+      - compress
+      - create
+      - extension .ndjson
+      - dateext
+      - dateyesterday
     /opt/so/log/elasticfleet/*_x_log:
       - daily
       - rotate 14

salt/manager/tools/sbin/soup

@@ -419,7 +419,8 @@ post_to_2.4.4() {
 }
 
 post_to_2.4.5() {
-  echo "Nothing to apply"
+  echo "Regenerating Elastic Agent Installers"
+  /sbin/so-elastic-agent-gen-installers
   POSTVERSION=2.4.5
 }
 
@@ -436,7 +437,7 @@ stop_salt_master() {
 
     echo ""
     echo "Storing salt-master pid."
-    MASTERPID=$(pgrep salt-master | head -1)
+    MASTERPID=$(pgrep -f '/opt/saltstack/salt/bin/python3.10 /usr/bin/salt-master MainProcess')
     echo "Found salt-master PID $MASTERPID"
     systemctl_func "stop" "salt-master"
     timeout 30 tail --pid=$MASTERPID -f /dev/null || echo "salt-master still running at $(date +"%T.%6N") after waiting 30s. We cannot kill due to systemd restart option."
@@ -455,7 +456,7 @@ stop_salt_minion() {
     set -e
 
     echo "Storing salt-minion pid."
-    MINIONPID=$(pgrep salt-minion | head -1)
+    MINIONPID=$(pgrep -f '/opt/saltstack/salt/bin/python3.10 /usr/bin/salt-minion' | head -1)
     echo "Found salt-minion PID $MINIONPID"
     systemctl_func "stop" "salt-minion"
 
@@ -859,7 +860,7 @@ main() {
     set +e
 
     echo "Checking the number of minions."
-    NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l)
+    NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | grep -v adv_ | wc -l)
     if [[ $UPGRADESALT -eq 1 ]] && [[ $NUM_MINIONS -gt 1 ]]; then
       if [[ $is_airgap -eq 0 ]]; then
         echo ""
@@ -875,9 +876,6 @@ main() {
     echo "Checking sudoers file."
     check_sudoers
 
-    echo "Checking for necessary user migrations."
-    so-user migrate
-
     systemctl_func "start" "$cron_service_name"
 
     if [[ -n $lsl_msg ]]; then

salt/nginx/etc/nginx.conf

@@ -296,7 +296,9 @@ http {
 		error_page 429 = @error429;
 
 		location @error401 {
-			add_header    Set-Cookie "AUTH_REDIRECT=$request_uri;Path=/;Max-Age=14400";
+			if ($request_uri ~* ^/(?!(^/api/.*))) {
+				add_header    Set-Cookie "AUTH_REDIRECT=$request_uri;Path=/;Max-Age=14400";
+			}
 			return        302 /auth/self-service/login/browser;
 		}
 

salt/sensor/soc_sensor.yaml

@@ -1,9 +1,9 @@
 sensor:
   interface:
     description: Main sensor monitoring interface.
-    helpLink: sensor.html
+    helpLink: network.html
     readonly: True
   mtu:
-    description: Main IP address of the grid host.
+    description: Maximum Transmission Unit (MTU) of the sensor monitoring interface.
-    helpLink: host.html
+    helpLink: network.html
     readonly: True

setup/so-setup

@@ -661,6 +661,7 @@ if ! [[ -f $install_opt_file ]]; then
 		logCmd "salt-call state.show_top"
 		sleep 2 # Debug RSA Key format errors
 		logCmd "salt-key -ya $MINION_ID"
+		logCmd "salt-call saltutil.sync_all"
 
 		logCmd "salt-call state.apply common.packages"
 		logCmd "salt-call state.apply common"
@@ -694,9 +695,11 @@ if ! [[ -f $install_opt_file ]]; then
 			logCmd "so-rule-update"
 			title "Downloading YARA rules"
 			logCmd "su socore -c '/usr/sbin/so-yara-download'"
-			if [[ $monints ]]; then
+			if [[ $monints || $is_import ]]; then
 				title "Restarting Suricata to pick up the new rules"
 				logCmd "so-suricata-restart"
+			fi
+			if [[ $monints ]]; then
 				title "Restarting Strelka to use new rules"
 				logCmd "so-strelka-restart"
 			fi

setup/so-verify

@@ -51,6 +51,7 @@ log_has_errors() {
         grep -vE "/nsm/rules/sigma*" | \
         grep -vE "/nsm/rules/yara*" | \
         grep -vE "Failed to restart snapd" | \
+        grep -vE "Login Failed Details" | \
         grep -vE "Running scope as unit" &> "$error_log"
     
     if [[ $? -eq 0 ]]; then