Merge remote-tracking branch 'origin/2.4/dev' into issue/10229

salt/docker/defaults.yaml

@@ -73,6 +73,7 @@ docker:
         - 80:80
         - 443:443
         - 8443:8443
+        - 7788:7788
       custom_bind_mounts: []
       extra_hosts: []
     'so-playbook':

salt/elasticfleet/install_agent_grid.sls

@@ -9,7 +9,7 @@
 
 run_installer:
   cmd.script:
-    - name: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux
+    - name: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64
     - cwd: /opt/so
     - args: -token={{ GRIDNODETOKEN }}
 

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers

@@ -4,20 +4,23 @@
 # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
 # this file except in compliance with the Elastic License 2.0.
 
-#so-elastic-agent-gen-installers $FleetHost $EnrollmentToken 
+#so-elastic-agent-gen-installers $FleetHostURLs $EnrollmentToken 
 
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 
 . /usr/sbin/so-common
 
+FLEETHOST="https://{{ GLOBALS.manager_ip }}:8220"
+
+for i in {1..30}
+do
    ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints")) | .api_key')
+   #FLEETHOST=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts' | jq -r  '.items[].host_urls[]' | paste -sd ',')
+if [[ $FLEETHOST ]] && [[  $ENROLLMENTOKEN ]]; then break; else sleep 10; fi
+done
+if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]]; then printf "\nFleet Host URL or Enrollment Token empty - exiting..." && exit; fi
 
-#FLEETHOST=$(lookup_pillar "server:url" "elasticfleet")
+OSARCH=( "linux-x86_64" "windows-x86_64" "darwin-x86_64" "darwin-aarch64" )
-FLEETHOST="{{ GLOBALS.manager_ip }}"
-
-#FLEETHOST=$1
-#ENROLLMENTOKEN=$2
-TARGETOS=( "linux" "darwin" "windows" )
 
 printf  "\n### Creating a temp directory at /nsm/elastic-agent-workspace\n"
 rm -rf /nsm/elastic-agent-workspace
@@ -25,9 +28,10 @@ mkdir -p /nsm/elastic-agent-workspace
 
 printf "\n### Extracting outer tarball and then each individual tarball/zip\n"
 tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-{{ GLOBALS.so_version }}.tar.gz -C /nsm/elastic-agent-workspace/
-unzip /nsm/elastic-agent-workspace/elastic-agent-*.zip -d /nsm/elastic-agent-workspace/
+unzip -q /nsm/elastic-agent-workspace/elastic-agent-*.zip -d /nsm/elastic-agent-workspace/
 for archive in /nsm/elastic-agent-workspace/*.tar.gz
 do
+  printf "\nExtracting $archive..."
   tar xf "$archive" -C /nsm/elastic-agent-workspace/
 done 
 
@@ -35,23 +39,27 @@ printf "\n### Stripping out unused components"
 find /nsm/elastic-agent-workspace/elastic-agent-*/data/elastic-agent-*/components -regex '.*fleet.*\|.*packet.*\|.*apm*.*\|.*audit.*\|.*heart.*\|.*cloud.*' -delete 
 
 printf "\n### Tarring everything up again"
-for OS in "${TARGETOS[@]}"
+for OS in "${OSARCH[@]}"
 do 
+  printf "\nCreating tarball for $OS..."
   rm -rf /nsm/elastic-agent-workspace/elastic-agent
-  mv /nsm/elastic-agent-workspace/elastic-agent-*-$OS-x86_64 /nsm/elastic-agent-workspace/elastic-agent
+  mv /nsm/elastic-agent-workspace/elastic-agent-*-$OS /nsm/elastic-agent-workspace/elastic-agent
-  tar -czvf /nsm/elastic-agent-workspace/$OS.tar.gz  -C /nsm/elastic-agent-workspace elastic-agent
+  tar -czf /nsm/elastic-agent-workspace/$OS.tar.gz  -C /nsm/elastic-agent-workspace elastic-agent
 done 
 
+GOTARGETOS=( "linux" "windows" "darwin" "darwin/arm64" )
+GOARCH="amd64"
 printf "\n### Generating OS packages using the cleaned up tarballs"
-for OS in "${TARGETOS[@]}"
+for GOOS in "${GOTARGETOS[@]}"
 do
-    printf "\n\n### Generating $OS Installer...\n"
+    if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin" && GOARCH="arm64"; fi
-    docker run -e CGO_ENABLED=0 -e GOOS=$OS \
+    printf "\n\n### Generating $GOOS/$GOARCH Installer...\n"
+    docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \
     --mount type=bind,source=/etc/ssl/certs/,target=/workspace/files/cert/ \
     --mount type=bind,source=/nsm/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \
     --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \
-    {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHost=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN"  -o /output/so-elastic-agent_$OS
+    {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN"  -o /output/so-elastic-agent_${GOOS}_${GOARCH}
-    printf "\n### $OS Installer Generated...\n"
+    printf "\n### $GOOS/$GOARCH Installer Generated...\n"
 done
 
 printf "\n### Cleaning up temp files in /nsm/elastic-agent-workspace"

salt/firewall/defaults.yaml

@@ -94,6 +94,10 @@ firewall:
       tcp:
         - 5601
       udp: []
+    localrules:
+      tcp: 
+        - 7788
+      udp: []
     mysql:
       tcp:
         - 3306
@@ -181,6 +185,7 @@ firewall:
                 - influxdb
                 - elasticsearch_rest
                 - elasticsearch_node
+                - localrules
             sensor:
               portgroups:
                 - beats_5044
@@ -364,6 +369,7 @@ firewall:
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
+                - localrules
             sensor:
               portgroups:
                 - beats_5044
@@ -501,6 +507,7 @@ firewall:
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
+                - localrules
             sensor:
               portgroups:
                 - beats_5044
@@ -648,6 +655,7 @@ firewall:
                 - elastic_agent_update
                 - endgame
                 - strelka_frontend
+                - localrules
             fleet:
               portgroups:
                 - elasticsearch_rest
@@ -1005,6 +1013,7 @@ firewall:
                 - elasticsearch_rest
                 - elasticsearch_node
                 - elastic_agent_control
+                - localrules
             sensor:
               portgroups:
                 - beats_5044

salt/firewall/soc_firewall.yaml

@@ -118,6 +118,9 @@ firewall:
     kibana:
       tcp: *tcpsettings
       udp: *udpsettings
+    localrules:
+      tcp: *tcpsettings
+      udp: *udpsettings
     mysql:
       tcp: *tcpsettings
       udp: *udpsettings

salt/idstools/etc/rulecat.conf

@@ -3,7 +3,7 @@
 {%- if GLOBALS.airgap is sameas true -%}
 --merged=/opt/so/rules/nids/all.rules
 --local=/opt/so/rules/nids/local.rules
-{%-   if GLOBAL.md_engine == "SURICATA" %}
+{%-   if GLOBALS.md_engine == "SURICATA" %}
 --local=/opt/so/rules/nids/sorules/extraction.rules
 --local=/opt/so/rules/nids/sorules/filters.rules
 {%-   endif %}

setup/so-functions

@@ -37,14 +37,8 @@ logCmd() {
 
 airgap_rules() {
 	# Copy the rules for suricata if using Airgap
-	mkdir -p /nsm/repo/rules
+	mkdir -p /nsm/rules
-	cp -v /root/SecurityOnion/agrules/emerging-all.rules /nsm/repo/rules/
+	cp -Rv /root/SecurityOnion/agrules/* /nsm/rules/
-	
-	# Copy over sigma rules
-	cp -Rv /root/SecurityOnion/agrules/sigma /nsm/repo/rules/
-
-	# Don't leave Strelka out
-	cp -Rv /root/SecurityOnion/agrules/strelka /nsm/repo/rules/
 }
 
 add_admin_user() {
@@ -973,7 +967,7 @@ download_elastic_agent_artifacts() {
 	else
 		logCmd "mkdir -p /nsm/elastic-fleet/artifacts/beats/elastic-agent/"
 		logCmd "curl --retry 5 --retry-delay 60 https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$SOVERSION.tar.gz --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz"
-		logCmd "tar -xf /nsm/elastic-fleet/artifacts/beats/elastic-agent_SO-$SOVERSION.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/"
+		logCmd "tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/"
 	fi
 }
 

setup/so-setup

@@ -202,12 +202,13 @@ if [ -n "$test_profile" ]; then
 
 	MINION_CIDR=10.0.0.0/8
 	MSRV=manager
-	if [[ "$test_profile" =~ "-net" ]]; then
+	if [[ "$test_profile" =~ "-net" ]] || [[ "$test_profile" =~ "-iso" ]]; then
 		address_type=DHCP
 	elif [[ "$test_profile" =~ "-cloud" ]]; then
 		MSRVIP=10.99.1.20
 	elif [[ "$test_profile" =~ "-airgap" ]]; then
 		is_airgap=true
+		address_type=DHCP
 	fi
 
 	if [ -f "/root/public_ip" ]; then
@@ -224,6 +225,8 @@ if [ -n "$test_profile" ]; then
 	WEBUSER=onionuser@somewhere.invalid
 	WEBPASSWD1=0n10nus3r
 	WEBPASSWD2=0n10nus3r
+
+	update_sudoers_for_testing
 fi
 
 # Make sure the setup type is suppoted.

setup/so-verify

@@ -44,6 +44,7 @@ log_has_errors() {
         grep -vE "Exception in callback None" | \
         grep -vE "deprecation: ERROR" | \
         grep -vE "code: 100" | \
+        grep -vE "/nsm/repo/rules/sigma/rules*" | \
         grep -vE "Running scope as unit" &> "$error_log"
     
     if [[ $? -eq 0 ]]; then