Merge remote-tracking branch 'origin/2.4/dev' into issue/10975

salt/common/packages.sls

@@ -21,7 +21,6 @@ commonpkgs:
       - python3-dateutil
       - python3-docker
       - python3-packaging
-      - python3-watchdog
       - python3-lxml
       - git
       - rsync
@@ -63,11 +62,7 @@ commonpkgs:
       - httpd-tools
       - jq
       - lvm2
-      {% if GLOBALS.os == 'CentOS Stream' %}
-      - MariaDB-devel
-      {% else %}
       - mariadb-devel
-      {% endif %}
       - net-tools
       - nmap-ncat
       - openssl
@@ -78,7 +73,6 @@ commonpkgs:
       - python3-packaging
       - python3-pyyaml
       - python3-rich
-      - python3-watchdog
       - rsync
       - sqlite
       - tcpdump

salt/common/soup_scripts.sls

@@ -19,4 +19,5 @@ soup_manager_scripts:
     - source: salt://manager/tools/sbin
     - include_pat:
         - so-firewall
+        - so-repo-sync
         - soup

salt/common/tools/sbin/so-common

@@ -446,6 +446,10 @@ set_os() {
 			OS=centos
 			OSVER=9
 			is_centos=true
+		elif grep -q "Oracle Linux Server release 9" /etc/system-release; then
+			OS=oel
+			OSVER=9
+			is_oracle=true
 		fi
 		cron_service_name="crond"
 	else

salt/common/tools/sbin/so-test

@@ -5,4 +5,10 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
+set -e
+
+# Playback live sample data onto monitor interface
 so-tcpreplay /opt/samples/* 2> /dev/null
+
+# Ingest sample pfsense log entry
+echo "<134>$(date '+%b %d %H:%M:%S') filterlog[31624]: 84,,,1567509287,igb0.244,match,pass,in,4,0x0,,64,0,0,DF,6,tcp,64,192.168.1.1,10.10.10.10,56320,443,0,S,3333585167,,65535,,mss;nop;wscale;nop;nop;TS;sackOK;eol" | nc -uv -w1 localhost 514 > /dev/null 2>&1

salt/elasticfleet/defaults.yaml

@@ -26,20 +26,51 @@ elasticfleet:
         - stderr
         - stdout
   packages:
+    - auditd
     - aws
     - azure
+    - barracuda
+    - cisco_asa
     - cloudflare
+    - crowdstrike
+    - darktrace
     - elasticsearch
     - endpoint
+    - f5_bigip
     - fleet_server
     - fim
+    - fortinet
+    - gcp
     - github
     - google_workspace
+    - http_endpoint
+    - httpjson
+    - juniper
+    - juniper_srx
+    - kafka_log
+    - lastpass
     - log
+    - m365_defender
+    - microsoft_defender_endpoint
+    - microsoft_dhcp
+    - netflow
+    - o365
+    - okta
     - osquery_manager
+    - panw
+    - pfsense
     - redis
+    - sentinel_one
+    - sonicwall_firewall
+    - symantec_endpoint
     - system
     - tcp
+    - ti_abusech
+    - ti_misp
+    - ti_otx
+    - ti_recordedfuture
     - udp
     - windows
+    - zscaler_zia
+    - zscaler_zpa
     - 1password

salt/elasticfleet/enabled.sls

@@ -68,11 +68,6 @@ so-elastic-fleet:
       - /etc/pki/elasticfleet-server.crt:/etc/pki/elasticfleet-server.crt:ro
       - /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro
       - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro
-      {% if GLOBALS.os_family == 'Debian' %}
-      - /etc/ssl/elasticfleet-server.crt:/etc/ssl/elasticfleet-server.crt:ro
-      - /etc/ssl/elasticfleet-server.key:/etc/ssl/elasticfleet-server.key:ro
-      - /etc/ssl/tls/certs/intca.crt:/etc/ssl/tls/certs/intca.crt:ro
-      {% endif %}
       - /opt/so/log/elasticfleet:/usr/share/elastic-agent/logs 
      {% if DOCKER.containers['so-elastic-fleet'].custom_bind_mounts %}
         {% for BIND in DOCKER.containers['so-elastic-fleet'].custom_bind_mounts %}
@@ -87,13 +82,8 @@ so-elastic-fleet:
       - FLEET_SERVER_POLICY_ID=FleetServer_{{ GLOBALS.hostname }}
       - FLEET_SERVER_CERT=/etc/pki/elasticfleet-server.crt
       - FLEET_SERVER_CERT_KEY=/etc/pki/elasticfleet-server.key
-      {% if GLOBALS.os_family == 'Debian' %}
-      - FLEET_CA=/etc/ssl/certs/intca.crt     
-      - FLEET_SERVER_ELASTICSEARCH_CA=/etc/ssl/certs/intca.crt
-      {% else %}
       - FLEET_CA=/etc/pki/tls/certs/intca.crt     
       - FLEET_SERVER_ELASTICSEARCH_CA=/etc/pki/tls/certs/intca.crt
-      {% endif %}
       - LOGS_PATH=logs
       {% if DOCKER.containers['so-elastic-fleet'].extra_env %}
         {% for XTRAENV in DOCKER.containers['so-elastic-fleet'].extra_env %}

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers

@@ -65,7 +65,7 @@ do
     if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin" && GOARCH="arm64"; fi
     printf "\n\n### Generating $GOOS/$GOARCH Installer...\n"
     docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \
-    --mount type=bind,source=/etc/ssl/certs/,target=/workspace/files/cert/ \
+    --mount type=bind,source=/etc/pki/tls/certs/,target=/workspace/files/cert/ \
     --mount type=bind,source=/nsm/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \
     --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \
     {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN"  -o /output/so-elastic-agent_${GOOS}_${GOARCH}

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup

@@ -6,11 +6,7 @@
 # this file except in compliance with the Elastic License 2.0.
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 
-{% if GLOBALS.os_family == 'Debian' %}
-INTCA=/etc/ssl/certs/intca.crt
-{% else %}
 INTCA=/etc/pki/tls/certs/intca.crt
-{% endif %}
 
 . /usr/sbin/so-elastic-fleet-common
 

salt/elasticsearch/enabled.sls

@@ -59,7 +59,7 @@ so-elasticsearch:
       {% if GLOBALS.is_manager %}
       - /etc/pki/ca.crt:/usr/share/elasticsearch/config/ca.crt:ro
       {% else %}
-      - /etc/ssl/certs/intca.crt:/usr/share/elasticsearch/config/ca.crt:ro
+      - /etc/pki/tls/certs/intca.crt:/usr/share/elasticsearch/config/ca.crt:ro
       {% endif %}
       - /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro
       - /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro

salt/elasticsearch/soc_elasticsearch.yaml

@@ -201,6 +201,7 @@ elasticsearch:
     so-logs-windows_x_powershell: *indexSettings
     so-logs-windows_x_powershell_operational: *indexSettings
     so-logs-windows_x_sysmon_operational: *indexSettings
+    so-logs-auditd_x_log: *indexSettings
     so-logs-aws_x_cloudtrail: *indexSettings
     so-logs-aws_x_cloudwatch_logs: *indexSettings
     so-logs-aws_x_ec2_logs: *indexSettings
@@ -221,9 +222,27 @@ elasticsearch:
     so-logs-azure_x_provisioning: *indexSettings
     so-logs-azure_x_signinlogs: *indexSettings
     so-logs-azure_x_springcloudlogs: *indexSettings
+    so-logs-barracuda_x_waf: *indexSettings
+    so-logs-cisco_asa_x_log: *indexSettings
     so-logs-cloudflare_x_audit: *indexSettings
     so-logs-cloudflare_x_logpull: *indexSettings
+    so-logs-crowdstrike_x_falcon: *indexSettings
+    so-logs-crowdstrike_x_fdr: *indexSettings
+    so-logs-darktrace_x_ai_analyst_alert: *indexSettings
+    so-logs-darktrace_x_model_breach_alert: *indexSettings
+    so-logs-darktrace_x_system_status_alert: *indexSettings
+    so-logs-f5_bigip_x_log: *indexSettings
     so-logs-fim_x_event: *indexSettings
+    so-logs-fortinet_x_clientendpoint: *indexSettings
+    so-logs-fortinet_x_firewall: *indexSettings
+    so-logs-fortinet_x_fortimail: *indexSettings
+    so-logs-fortinet_x_fortimanager: *indexSettings
+    so-logs-fortinet_x_fortigate: *indexSettings
+    so-logs-gcp_x_audit: *indexSettings
+    so-logs-gcp_x_dns: *indexSettings
+    so-logs-gcp_x_firewall: *indexSettings
+    so-logs-gcp_x_loadbalancing_logs: *indexSettings
+    so-logs-gcp_x_vpcflow: *indexSettings
     so-logs-github_x_audit: *indexSettings
     so-logs-github_x_code_scanning: *indexSettings
     so-logs-github_x_dependabot: *indexSettings
@@ -243,6 +262,52 @@ elasticsearch:
     so-logs-google_workspace_x_saml: *indexSettings
     so-logs-google_workspace_x_token: *indexSettings
     so-logs-google_workspace_x_user_accounts: *indexSettings
+    so-logs-http_endpoint_x_generic: *indexSettings
+    so-logs-httpjson_x_generic: *indexSettings
+    so-logs-juniper_x_junos: *indexSettings
+    so-logs-juniper_x_netscreen: *indexSettings
+    so-logs-juniper_x_srx: *indexSettings
+    so-logs-juniper_srx_x_log: *indexSettings
+    so-logs-kafka_log_x_generic: *indexSettings
+    so-logs-lastpass_x_detailed_shared_folder: *indexSettings
+    so-logs-lastpass_x_event_report: *indexSettings
+    so-logs-lastpass_x_user: *indexSettings
+    so-logs-m365_defender_x_event: *indexSettings
+    so-logs-m365_defender_x_incident: *indexSettings
+    so-logs-m365_defender_x_log: *indexSettings
+    so-logs-microsoft_defender_endpoint_x_log: *indexSettings
+    so-logs-microsoft_dhcp_x_log: *indexSettings
+    so-logs-netflow_x_log: *indexSettings
+    so-logs-o365_x_audit: *indexSettings
+    so-logs-okta_x_system: *indexSettings
+    so-logs-panw_x_panos: *indexSettings
+    so-logs-pfsense_x_log: *indexSettings
+    so-logs-sentinel_one_x_activity: *indexSettings
+    so-logs-sentinel_one_x_agent: *indexSettings
+    so-logs-sentinel_one_x_alert: *indexSettings
+    so-logs-sentinel_one_x_group: *indexSettings
+    so-logs-sentinel_one_x_threat: *indexSettings
+    so-logs-sonicwall_firewall_x_log: *indexSettings
+    so-logs-symantec_endpoint_x_log: *indexSettings
+    so-logs-ti_abusech_x_malware: *indexSettings
+    so-logs-ti_abusech_x_malwarebazaar: *indexSettings
+    so-logs-ti_abusech_x_threatfox: *indexSettings
+    so-logs-ti_abusech_x_url: *indexSettings
+    so-logs-ti_misp_x_threat: *indexSettings
+    so-logs-ti_misp_x_threat_attributes: *indexSettings
+    so-logs-ti_otx_x_threat: *indexSettings
+    so-logs-ti_recordedfuture_x_latest_ioc-template: *indexSettings
+    so-logs-ti_recordedfuture_x_threat: *indexSettings
+    so-logs-zscaler_zia_x_alerts: *indexSettings
+    so-logs-zscaler_zia_x_dns: *indexSettings
+    so-logs-zscaler_zia_x_firewall: *indexSettings
+    so-logs-zscaler_zia_x_tunnel: *indexSettings
+    so-logs-zscaler_zia_x_web: *indexSettings
+    so-logs-zscaler_zpa_x_app_connector_status: *indexSettings
+    so-logs-zscaler_zpa_x_audit: *indexSettings
+    so-logs-zscaler_zpa_x_browser_access: *indexSettings
+    so-logs-zscaler_zpa_x_user_activity: *indexSettings
+    so-logs-zscaler_zpa_x_user_status: *indexSettings
     so-logs-1password_x_item_usages: *indexSettings
     so-logs-1password_x_signin_attempts: *indexSettings
     so-logs-osquery-manager-actions: *indexSettings

salt/logstash/enabled.sls

@@ -73,7 +73,7 @@ so-logstash:
       {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import'] %}
       - /etc/pki/ca.crt:/usr/share/filebeat/ca.crt:ro
       {% else %}
-      - /etc/ssl/certs/intca.crt:/usr/share/filebeat/ca.crt:ro
+      - /etc/pki/tls/certs/intca.crt:/usr/share/filebeat/ca.crt:ro
       {% endif %}
       {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-searchnode'] %}
       - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro

salt/manager/tools/sbin/so-repo-sync

@@ -11,6 +11,8 @@ set_version
 set_os
 salt_minion_count
 
+set -e
+
 curl --retry 5 --retry-delay 60 -A "reposync/$VERSION/$OS/$(uname -r)/$MINIONCOUNT" https://sigs.securityonion.net/checkup --output /tmp/checkup
 dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/
 createrepo /nsm/repo

salt/manager/tools/sbin/soup

@@ -303,6 +303,7 @@ check_log_size_limit() {
 
 check_os_updates() {
   # Check to see if there are OS updates
+  echo "Checking for OS updates."
   NEEDUPDATES="We have detected missing operating system (OS) updates. Do you want to install these OS updates now? This could take a while depending on the size of your grid and how many packages are missing, but it is recommended to keep your system updated."
   OSUPDATES=$(dnf -q list updates | grep -v docker | grep -v containerd | grep -v salt | grep -v Available | wc -l)
   if [[ "$OSUPDATES" -gt 0 ]]; then
@@ -437,6 +438,11 @@ post_to_2.4.20() {
   POSTVERSION=2.4.20
 }
 
+repo_sync() {
+  echo "Sync the local repo."
+  su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync."
+}
+
 stop_salt_master() {
     # kill all salt jobs across the grid because the hang indefinitely if they are queued and salt-master restarts
     set +e
@@ -762,8 +768,6 @@ main() {
   fi
   echo "Verifying we have the latest soup script."
   verify_latest_update_script
-  echo "Checking for OS updates."
-  check_os_updates
 
   echo "Let's see if we need to update Security Onion."
   upgrade_check
@@ -776,6 +780,10 @@ main() {
   if [[ $is_airgap -eq 0 ]]; then
     yum clean all
     check_os_updates
+  elif [[ $OS == 'oel' ]]; then
+    # sync remote repo down to local if not airgap
+    repo_sync
+    check_os_updates
   fi
 
   if [ "$is_hotfix" == "true" ]; then

salt/redis/enabled.sls

@@ -33,7 +33,7 @@ so-redis:
       {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import'] %}
       - /etc/pki/ca.crt:/certs/ca.crt:ro
       {% else %}
-      - /etc/ssl/certs/intca.crt:/certs/ca.crt:ro
+      - /etc/pki/tls/certs/intca.crt:/certs/ca.crt:ro
       {% endif %}
       {% if DOCKER.containers['so-redis'].custom_bind_mounts %}
         {% for BIND in DOCKER.containers['so-redis'].custom_bind_mounts %}

salt/soc/defaults.map.jinja

@@ -16,7 +16,7 @@
 {# add nodes from the logstash:nodes pillar to soc.server.modules.elastic.remoteHostUrls #}
 {% for node_type, minions in salt['pillar.get']('logstash:nodes', {}).items() %}
 {%   for m in minions.keys() %}
-{%     do SOCDEFAULTS.soc.config.server.modules.elastic.remoteHostUrls.append(m) %}
+{%     do SOCDEFAULTS.soc.config.server.modules.elastic.remoteHostUrls.append('https://' ~ m ~ ':9200') %}
 {%   endfor %}
 {% endfor %}
 

salt/soc/defaults.yaml

@@ -474,19 +474,6 @@ soc:
         - event.dataset
         - process.executable
         - user.name
-      ':ossec:':
-        - soc_timestamp
-        - source.ip
-        - source.port
-        - destination.ip
-        - destination.port
-        - rule.name
-        - rule.level
-        - rule.category
-        - process.name
-        - user.name
-        - user.escalated
-        - location
       ':strelka:file':
         - soc_timestamp
         - file.name
@@ -523,28 +510,6 @@ soc:
         - message
         - kibana.log.meta.req.headers.x-real-ip
         - event.dataset
-      '::rootcheck':
-        - soc_timestamp
-        - host.name
-        - metadata.ip_address
-        - log.full
-        - event.dataset
-        - event.module
-      '::ossec':
-        - soc_timestamp
-        - host.name
-        - metadata.ip_address
-        - log.full
-        - event.dataset
-        - event.module
-      '::syscollector':
-        - soc_timestamp
-        - host.name
-        - metadata.ip_address
-        - wazuh.data.type
-        - log.full
-        - event.dataset
-        - event.module
       ':syslog:syslog':
         - soc_timestamp
         - host.name
@@ -1621,21 +1586,15 @@ soc:
               - rule.uuid
               - rule.category
               - rule.rev
-            ':ossec:':
+            ':playbook:':
               - soc_timestamp
               - rule.name
               - event.severity_label
-              - source.ip
+              - event_data.event.module
-              - source.port
+              - event_data.event.category
-              - destination.ip
+              - event_data.process.executable
-              - destination.port
+              - event_data.process.pid
-              - rule.level
+              - event_data.winlog.computer_name
-              - rule.category
-              - process.name
-              - user.name
-              - user.escalated
-              - location
-              - process.name
           queryBaseFilter: tags:alert
           queryToggleFilters:
             - name: acknowledged

salt/ssl/init.sls

@@ -36,14 +36,24 @@ include:
     {% set ca_server = global_ca_server[0] %}
 {% endif %}
 
-
+cacertdir:
+  file.directory:
+    - name: /etc/pki/tls/certs
+    - makedirs: True
 
 # Trust the CA
 trusttheca:
   x509.pem_managed:
-    - name: /etc/ssl/certs/intca.crt
+    - name: /etc/pki/tls/certs/intca.crt
     - text:  {{ trusttheca_text }}
 
+{% if GLOBALS.os_family == 'Debian' %}
+symlinkca:
+  file.symlink:
+    - target: /etc/pki/tls/certs/intca.crt
+    - name: /etc/ssl/certs/intca.crt
+{% endif %}
+
 # Install packages needed for the sensor
 m2cryptopkgs:
   pkg.installed:

salt/ssl/remove.sls

@@ -1,6 +1,6 @@
 trusttheca:
   file.absent:
-    - name: /etc/ssl/certs/intca.crt
+    - name: /etc/pki/tls/certs/intca.crt
 
 influxdb_key:
   file.absent:

salt/strelka/filestream/config.sls

@@ -47,6 +47,21 @@ filestream_config:
         FILESTREAMCONFIG: {{ STRELKAMERGED.filestream.config }}
 
 # Filecheck Section
+{% if GLOBALS.os_family == 'Debian' %}
+install_watchdog:
+  pkg.installed:
+    - name: python3-watchdog
+
+{% elif GLOBALS.os_family == 'RedHat' %}
+remove_old_watchdog:
+  pkg.removed:
+    - name: python3-watchdog
+
+install_watchdog:
+  pkg.installed:
+    - name: securityonion-python39-watchdog
+{% endif %}
+
 filecheck_logdir:
   file.directory:
     - name: /opt/so/log/strelka
@@ -127,6 +142,7 @@ filecheck_restart:
     - onchanges:
       - file: filecheck_script
       - file: filecheck_conf
+      - pkg: install_watchdog
 
 filcheck_history_clean:
   cron.present:

salt/telegraf/enabled.sls

@@ -46,7 +46,7 @@ so-telegraf:
       {% if GLOBALS.role in ['so-manager', 'so-eval', 'so-managersearch' ] %}
       - /etc/pki/ca.crt:/etc/telegraf/ca.crt:ro
       {% else %}
-      - /etc/ssl/certs/intca.crt:/etc/telegraf/ca.crt:ro
+      - /etc/pki/tls/certs/intca.crt:/etc/telegraf/ca.crt:ro
       {% endif %}
       - /etc/pki/influxdb.crt:/etc/telegraf/telegraf.crt:ro
       - /etc/pki/influxdb.key:/etc/telegraf/telegraf.key:ro

setup/so-setup

@@ -69,7 +69,7 @@ detect_os
 # Ubuntu/Debian whiptail pallete to make it look the same as CentOS and Rocky.
 set_palette >> $setup_log 2>&1
 
-if [[ $not_supported ]]; then
+if [[ $not_supported ]] && [ -z "$test_profile" ]; then
 	if [[ "$OSVER" == "focal" ]]; then
 		if (whiptail_focal_warning); then
 			true