Merge remote-tracking branch 'origin/2.4/dev' into issue/10229

2025-12-06 09:12:45 +01:00 · 2023-05-16 09:22:48 -04:00
parent a0e08e4f41 1724565331
commit b253cd45ca
9 changed files with 49 additions and 30 deletions
--- a/salt/docker/defaults.yaml
+++ b/salt/docker/defaults.yaml
@@ -73,6 +73,7 @@ docker:
        - 80:80
        - 443:443
        - 8443:8443
+        - 7788:7788
      custom_bind_mounts: []
      extra_hosts: []
    'so-playbook':
--- a/salt/elasticfleet/install_agent_grid.sls
+++ b/salt/elasticfleet/install_agent_grid.sls
@@ -9,7 +9,7 @@

 run_installer:
  cmd.script:
-    - name: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux
+    - name: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64
    - cwd: /opt/so
    - args: -token={{ GRIDNODETOKEN }}

--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers
@@ -4,20 +4,23 @@
 # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
 # this file except in compliance with the Elastic License 2.0.

-#so-elastic-agent-gen-installers $FleetHost $EnrollmentToken 
+#so-elastic-agent-gen-installers $FleetHostURLs $EnrollmentToken 

 {% from 'vars/globals.map.jinja' import GLOBALS %}

 . /usr/sbin/so-common

+FLEETHOST="https://{{ GLOBALS.manager_ip }}:8220"
+
+for i in {1..30}
+do
   ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints")) | .api_key')
+   #FLEETHOST=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts' | jq -r  '.items[].host_urls[]' | paste -sd ',')
+if [[ $FLEETHOST ]] && [[  $ENROLLMENTOKEN ]]; then break; else sleep 10; fi
+done
+if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]]; then printf "\nFleet Host URL or Enrollment Token empty - exiting..." && exit; fi

-#FLEETHOST=$(lookup_pillar "server:url" "elasticfleet")
-FLEETHOST="{{ GLOBALS.manager_ip }}"
-
-#FLEETHOST=$1
-#ENROLLMENTOKEN=$2
-TARGETOS=( "linux" "darwin" "windows" )
+OSARCH=( "linux-x86_64" "windows-x86_64" "darwin-x86_64" "darwin-aarch64" )

 printf  "\n### Creating a temp directory at /nsm/elastic-agent-workspace\n"
 rm -rf /nsm/elastic-agent-workspace
@@ -25,9 +28,10 @@ mkdir -p /nsm/elastic-agent-workspace

 printf "\n### Extracting outer tarball and then each individual tarball/zip\n"
 tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-{{ GLOBALS.so_version }}.tar.gz -C /nsm/elastic-agent-workspace/
-unzip /nsm/elastic-agent-workspace/elastic-agent-*.zip -d /nsm/elastic-agent-workspace/
+unzip -q /nsm/elastic-agent-workspace/elastic-agent-*.zip -d /nsm/elastic-agent-workspace/
 for archive in /nsm/elastic-agent-workspace/*.tar.gz
 do
+  printf "\nExtracting $archive..."
  tar xf "$archive" -C /nsm/elastic-agent-workspace/
 done 

@@ -35,23 +39,27 @@ printf "\n### Stripping out unused components"
 find /nsm/elastic-agent-workspace/elastic-agent-*/data/elastic-agent-*/components -regex '.*fleet.*\|.*packet.*\|.*apm*.*\|.*audit.*\|.*heart.*\|.*cloud.*' -delete 

 printf "\n### Tarring everything up again"
-for OS in "${TARGETOS[@]}"
+for OS in "${OSARCH[@]}"
 do 
+  printf "\nCreating tarball for $OS..."
  rm -rf /nsm/elastic-agent-workspace/elastic-agent
-  mv /nsm/elastic-agent-workspace/elastic-agent-*-$OS-x86_64 /nsm/elastic-agent-workspace/elastic-agent
-  tar -czvf /nsm/elastic-agent-workspace/$OS.tar.gz  -C /nsm/elastic-agent-workspace elastic-agent
+  mv /nsm/elastic-agent-workspace/elastic-agent-*-$OS /nsm/elastic-agent-workspace/elastic-agent
+  tar -czf /nsm/elastic-agent-workspace/$OS.tar.gz  -C /nsm/elastic-agent-workspace elastic-agent
 done 

+GOTARGETOS=( "linux" "windows" "darwin" "darwin/arm64" )
+GOARCH="amd64"
 printf "\n### Generating OS packages using the cleaned up tarballs"
-for OS in "${TARGETOS[@]}"
+for GOOS in "${GOTARGETOS[@]}"
 do
-    printf "\n\n### Generating $OS Installer...\n"
-    docker run -e CGO_ENABLED=0 -e GOOS=$OS \
+    if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin" && GOARCH="arm64"; fi
+    printf "\n\n### Generating $GOOS/$GOARCH Installer...\n"
+    docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \
    --mount type=bind,source=/etc/ssl/certs/,target=/workspace/files/cert/ \
    --mount type=bind,source=/nsm/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \
    --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \
-    {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHost=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN"  -o /output/so-elastic-agent_$OS
-    printf "\n### $OS Installer Generated...\n"
+    {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN"  -o /output/so-elastic-agent_${GOOS}_${GOARCH}
+    printf "\n### $GOOS/$GOARCH Installer Generated...\n"
 done

 printf "\n### Cleaning up temp files in /nsm/elastic-agent-workspace"
--- a/salt/firewall/defaults.yaml
+++ b/salt/firewall/defaults.yaml
@@ -94,6 +94,10 @@ firewall:
      tcp:
        - 5601
      udp: []
+    localrules:
+      tcp: 
+        - 7788
+      udp: []
    mysql:
      tcp:
        - 3306
@@ -181,6 +185,7 @@ firewall:
                - influxdb
                - elasticsearch_rest
                - elasticsearch_node
+                - localrules
            sensor:
              portgroups:
                - beats_5044
@@ -364,6 +369,7 @@ firewall:
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
+                - localrules
            sensor:
              portgroups:
                - beats_5044
@@ -501,6 +507,7 @@ firewall:
                - elastic_agent_control
                - elastic_agent_data
                - elastic_agent_update
+                - localrules
            sensor:
              portgroups:
                - beats_5044
@@ -648,6 +655,7 @@ firewall:
                - elastic_agent_update
                - endgame
                - strelka_frontend
+                - localrules
            fleet:
              portgroups:
                - elasticsearch_rest
@@ -1005,6 +1013,7 @@ firewall:
                - elasticsearch_rest
                - elasticsearch_node
                - elastic_agent_control
+                - localrules
            sensor:
              portgroups:
                - beats_5044
--- a/salt/firewall/soc_firewall.yaml
+++ b/salt/firewall/soc_firewall.yaml
@@ -118,6 +118,9 @@ firewall:
    kibana:
      tcp: *tcpsettings
      udp: *udpsettings
+    localrules:
+      tcp: *tcpsettings
+      udp: *udpsettings
    mysql:
      tcp: *tcpsettings
      udp: *udpsettings
--- a/salt/idstools/etc/rulecat.conf
+++ b/salt/idstools/etc/rulecat.conf
@@ -3,7 +3,7 @@
 {%- if GLOBALS.airgap is sameas true -%}
 --merged=/opt/so/rules/nids/all.rules
 --local=/opt/so/rules/nids/local.rules
-{%-   if GLOBAL.md_engine == "SURICATA" %}
+{%-   if GLOBALS.md_engine == "SURICATA" %}
 --local=/opt/so/rules/nids/sorules/extraction.rules
 --local=/opt/so/rules/nids/sorules/filters.rules
 {%-   endif %}
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -37,14 +37,8 @@ logCmd() {

 airgap_rules() {
 	# Copy the rules for suricata if using Airgap
-	mkdir -p /nsm/repo/rules
-	cp -v /root/SecurityOnion/agrules/emerging-all.rules /nsm/repo/rules/
-	
-	# Copy over sigma rules
-	cp -Rv /root/SecurityOnion/agrules/sigma /nsm/repo/rules/
-
-	# Don't leave Strelka out
-	cp -Rv /root/SecurityOnion/agrules/strelka /nsm/repo/rules/
+	mkdir -p /nsm/rules
+	cp -Rv /root/SecurityOnion/agrules/* /nsm/rules/
 }

 add_admin_user() {
@@ -973,7 +967,7 @@ download_elastic_agent_artifacts() {
 	else
 		logCmd "mkdir -p /nsm/elastic-fleet/artifacts/beats/elastic-agent/"
 		logCmd "curl --retry 5 --retry-delay 60 https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$SOVERSION.tar.gz --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz"
-		logCmd "tar -xf /nsm/elastic-fleet/artifacts/beats/elastic-agent_SO-$SOVERSION.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/"
+		logCmd "tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/"
 	fi
 }

--- a/setup/so-setup
+++ b/setup/so-setup
@@ -202,12 +202,13 @@ if [ -n "$test_profile" ]; then

 	MINION_CIDR=10.0.0.0/8
 	MSRV=manager
-	if [[ "$test_profile" =~ "-net" ]]; then
+	if [[ "$test_profile" =~ "-net" ]] || [[ "$test_profile" =~ "-iso" ]]; then
 		address_type=DHCP
 	elif [[ "$test_profile" =~ "-cloud" ]]; then
 		MSRVIP=10.99.1.20
 	elif [[ "$test_profile" =~ "-airgap" ]]; then
 		is_airgap=true
+		address_type=DHCP
 	fi

 	if [ -f "/root/public_ip" ]; then
@@ -224,6 +225,8 @@ if [ -n "$test_profile" ]; then
 	WEBUSER=onionuser@somewhere.invalid
 	WEBPASSWD1=0n10nus3r
 	WEBPASSWD2=0n10nus3r
+
+	update_sudoers_for_testing
 fi

 # Make sure the setup type is suppoted.
--- a/setup/so-verify
+++ b/setup/so-verify
@@ -44,6 +44,7 @@ log_has_errors() {
        grep -vE "Exception in callback None" | \
        grep -vE "deprecation: ERROR" | \
        grep -vE "code: 100" | \
+        grep -vE "/nsm/repo/rules/sigma/rules*" | \
        grep -vE "Running scope as unit" &> "$error_log"
    
    if [[ $? -eq 0 ]]; then