Merge branch '2.4/dev' into vlb2

salt/elasticfleet/config.sls

@@ -63,6 +63,14 @@ eastatedir:
     - group: 939
     - makedirs: True
 
+custommappingsdir:
+  file.directory:
+    - name: /nsm/custom-mappings
+    - user: 947
+    - group: 939
+    - makedirs: True
+
+
 eapackageupgrade:
   file.managed:
     - name: /usr/sbin/so-elastic-fleet-package-upgrade
@@ -74,13 +82,6 @@ eapackageupgrade:
 
 {%   if GLOBALS.role != "so-fleet" %}
    
-soresourcesrepoconfig:
-  git.config_set:
-    - name: safe.directory
-    - value: /nsm/securityonion-resources
-    - global: True
-    - user: socore
-    
 {% if not GLOBALS.airgap %}
 soresourcesrepoclone:
   git.latest:

salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json

@@ -0,0 +1,35 @@
+{
+  "package": {
+    "name": "log",
+    "version": ""
+  },
+  "name": "so-ip-mappings",
+  "namespace": "so",
+  "description": "IP Description mappings",
+  "policy_id": "so-grid-nodes_general",
+  "vars": {},
+  "inputs": {
+    "logs-logfile": {
+      "enabled": true,
+      "streams": {
+        "log.logs": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/nsm/custom-mappings/ip-descriptions.csv"
+            ],
+            "data_stream.dataset": "hostnamemappings",
+            "tags": [
+              "so-ip-mappings"
+            ],
+            "processors": "- decode_csv_fields:\n    fields:\n      message: decoded.csv\n    separator: \",\"\n    ignore_missing: false\n    overwrite_keys: true\n    trim_leading_space: true\n    fail_on_error: true\n\n- extract_array:\n    field: decoded.csv\n    mappings:\n      so.ip_address: '0'\n      so.description: '1'\n\n- script:\n    lang: javascript\n    source: >\n      function process(event) {\n          var ip = event.Get('so.ip_address');\n          var validIpRegex = /^((25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)\\.){3}(25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)$/\n          if (!validIpRegex.test(ip)) {\n              event.Cancel();\n          }\n      }\n- fingerprint:\n    fields: [\"so.ip_address\"]\n    target_field: \"@metadata._id\"\n",
+            "custom": ""
+          }
+        }
+      }
+    }
+  },
+  "force": true
+}
+
+

salt/elasticsearch/defaults.yaml

@@ -599,6 +599,35 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
+    so-ip-mappings:
+      index_sorting: false
+      index_template:
+        composed_of:
+        - so-ip-mappings
+        ignore_missing_component_templates: []
+        index_patterns:
+        - so-ip*
+        priority: 500
+        template:
+          mappings:
+            date_detection: false
+            dynamic_templates:
+            - strings_as_keyword:
+                mapping:
+                  ignore_above: 1024
+                  type: keyword
+                match_mapping_type: string
+          settings:
+            index:
+              mapping:
+                total_fields:
+                  limit: 1500
+              number_of_replicas: 0
+              number_of_shards: 1
+              refresh_interval: 30s
+              sort:
+                field: '@timestamp'
+                order: desc
     so-items:
       index_sorting: false
       index_template:
@@ -3470,28 +3499,70 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
-    so-logs-crowdstrike_x_falcon:
+    so-logs-crowdstrike_x_alert:
-      index_sorting: false
+      index_sorting: False
       index_template:
+        index_patterns:
+        - logs-crowdstrike.alert-*
+        template:
+          settings:
+            index:
+              number_of_replicas: 0
+        composed_of:
+        - logs-crowdstrike.alert@package
+        - logs-crowdstrike.alert@custom
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+        ignore_missing_component_templates:
+        - logs-crowdstrike.alert@custom
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-logs-crowdstrike_x_falcon:
+      index_sorting: False
+      index_template:
+        index_patterns:
+        - logs-crowdstrike.falcon-*
+        template:
+          settings:
+            index:
+              number_of_replicas: 0
         composed_of:
         - logs-crowdstrike.falcon@package
         - logs-crowdstrike.falcon@custom
         - so-fleet_globals-1
         - so-fleet_agent_id_verification-1
+        priority: 501
         data_stream:
-          allow_custom_routing: false
           hidden: false
+          allow_custom_routing: false
         ignore_missing_component_templates:
         - logs-crowdstrike.falcon@custom
-        index_patterns:
-        - logs-crowdstrike.falcon-*
-        priority: 501
-        template:
-          settings:
-            index:
-              lifecycle:
-                name: so-logs-crowdstrike.falcon-logs
-              number_of_replicas: 0
       policy:
         phases:
           cold:
@@ -3517,27 +3588,69 @@ elasticsearch:
                 priority: 50
             min_age: 30d
     so-logs-crowdstrike_x_fdr:
-      index_sorting: false
+      index_sorting: False
       index_template:
+        index_patterns:
+        - logs-crowdstrike.fdr-*
+        template:
+          settings:
+            index:
+              number_of_replicas: 0
         composed_of:
         - logs-crowdstrike.fdr@package
         - logs-crowdstrike.fdr@custom
         - so-fleet_globals-1
         - so-fleet_agent_id_verification-1
+        priority: 501
         data_stream:
-          allow_custom_routing: false
           hidden: false
+          allow_custom_routing: false
         ignore_missing_component_templates:
         - logs-crowdstrike.fdr@custom
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-logs-crowdstrike_x_host:
+      index_sorting: False
+      index_template:
         index_patterns:
-        - logs-crowdstrike.fdr-*
+        - logs-crowdstrike.host-*
-        priority: 501
         template:
           settings:
             index:
-              lifecycle:
-                name: so-logs-crowdstrike.fdr-logs
               number_of_replicas: 0
+        composed_of:
+        - logs-crowdstrike.host@package
+        - logs-crowdstrike.host@custom
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+        ignore_missing_component_templates:
+        - logs-crowdstrike.host@custom
       policy:
         phases:
           cold:

salt/elasticsearch/soc_elasticsearch.yaml

@@ -396,8 +396,10 @@ elasticsearch:
     so-logs-citrix_waf_x_log: *indexSettings
     so-logs-cloudflare_x_audit: *indexSettings
     so-logs-cloudflare_x_logpull: *indexSettings
+    so-logs-crowdstrike_x_alert: *indexSettings
     so-logs-crowdstrike_x_falcon: *indexSettings
     so-logs-crowdstrike_x_fdr: *indexSettings
+    so-logs-crowdstrike_x_host: *indexSettings
     so-logs-darktrace_x_ai_analyst_alert: *indexSettings
     so-logs-darktrace_x_model_breach_alert: *indexSettings
     so-logs-darktrace_x_system_status_alert: *indexSettings

salt/elasticsearch/templates/component/elastic-agent/logs-crowdstrike.alert@custom.json

@@ -0,0 +1,36 @@
+{
+  "template": {
+    "mappings": {
+      "properties": {
+        "host": {
+	  "properties":{
+            "ip": {
+              "type": "ip"
+            }
+	  }
+	},
+	"related": {
+          "properties":{
+            "ip": {
+              "type": "ip"
+            }
+          }
+        },
+	"destination": {
+          "properties":{
+            "ip": {
+              "type": "ip"
+            }
+          }
+        },
+	"source": {
+          "properties":{
+            "ip": {
+              "type": "ip"
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/elastic-agent/logs-crowdstrike.host@custom.json

@@ -0,0 +1,36 @@
+{
+  "template": {
+    "mappings": {
+      "properties": {
+        "host": {
+	  "properties":{
+            "ip": {
+              "type": "ip"
+            }
+	  }
+	},
+	"related": {
+          "properties":{
+            "ip": {
+              "type": "ip"
+            }
+          }
+        },
+	"destination": {
+          "properties":{
+            "ip": {
+              "type": "ip"
+            }
+          }
+        },
+	"source": {
+          "properties":{
+            "ip": {
+              "type": "ip"
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/so/detection-mappings.json

@@ -82,6 +82,12 @@
               "ignore_above": 1024,
               "type": "keyword"
             },
+            "sourceCreated": {
+              "type": "date"
+            },
+            "sourceUpdated": {
+              "type": "date"
+            },
             "overrides": {
               "properties": {
                 "type": {

salt/elasticsearch/templates/component/so/so-ip-mappings.json

@@ -0,0 +1,25 @@
+{
+  "_meta": {
+    "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-network.html",
+    "ecs_version": "1.12.2"
+  },
+  "template": {
+    "mappings": {
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "so": {
+          "properties": {
+            "ip_address": {
+              "type": "ip"
+            },
+            "description": {
+              "type": "text"
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/logstash/pipelines/config/so/9805_output_elastic_agent.conf.jinja

@@ -1,4 +1,18 @@
 output {
+  if "elastic-agent" in [tags] and "so-ip-mappings" in [tags] {
+    elasticsearch {
+      hosts => "{{ GLOBALS.hostname }}"
+      data_stream => false
+      user => "{{ ES_USER }}"
+      password => "{{ ES_PASS }}"
+      document_id => "%{[metadata][_id]}"
+      index => "so-ip-mappings"
+      silence_errors_in_log => ["version_conflict_engine_exception"]
+      ssl => true
+      ssl_certificate_verification => false
+    }
+  }
+  else {
     if "elastic-agent" in [tags] {
       if [metadata][pipeline] {
         if [metadata][_id] {
@@ -40,4 +54,5 @@ output {
         }
       }
     }
+  }
 }

salt/manager/init.sls

@@ -6,10 +6,7 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   from 'strelka/map.jinja' import STRELKAMERGED %}
+{%   from 'manager/map.jinja' import MANAGERMERGED %}
-{%   import_yaml 'manager/defaults.yaml' as MANAGERDEFAULTS %}
-{%   set MANAGERMERGED = salt['pillar.get']('manager', MANAGERDEFAULTS.manager, merge=true) %}
-{%   from 'strelka/map.jinja' import STRELKAMERGED %}
 
 include:
   - salt.minion
@@ -141,6 +138,16 @@ rules_dir:
     - group: socore
     - makedirs: True
 
+git_config_set_safe_dirs:
+  git.config_set:
+    - name: safe.directory
+    - global: True
+    - user: socore
+    - multivar:
+      - /nsm/rules/custom-local-repos/local-sigma
+      - /nsm/rules/custom-local-repos/local-yara
+      - /nsm/securityonion-resources
+      - /opt/so/conf/soc/ai_summary_repos/securityonion-resources
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/manager/map.jinja

@@ -5,3 +5,7 @@
 
 {% import_yaml 'manager/defaults.yaml' as MANAGERDEFAULTS %}
 {% set MANAGERMERGED = salt['pillar.get']('manager', MANAGERDEFAULTS.manager, merge=True) %}
+
+{% if grains.os != 'OEL' %}
+{%   do MANAGERMERGED.reposync.update({'enabled': False}) %}
+{% endif %}

salt/manager/tools/sbin/soup

@@ -719,6 +719,9 @@ up_to_2.4.120() {
   mkdir /opt/so/saltstack/local/pillar/versionlock
   touch /opt/so/saltstack/local/pillar/versionlock/adv_versionlock.sls /opt/so/saltstack/local/pillar/versionlock/soc_versionlock.sls
 
+  # New Grid Integration added this release
+  rm -f /opt/so/state/eaintegrations.txt
+
   INSTALLEDVERSION=2.4.120
 }
 

salt/soc/config.sls

@@ -198,6 +198,49 @@ socsensoronirepos:
     - mode: 775
     - makedirs: True
 
+
+create_custom_local_yara_repo_template:
+  git.present:
+    - name: /nsm/rules/custom-local-repos/local-yara
+    - bare: False
+    - force: True
+
+add_readme_custom_local_yara_repo_template:
+  file.managed:
+    - name: /nsm/rules/custom-local-repos/local-yara/README
+    - source: salt://soc/files/soc/detections_custom_repo_template_readme.jinja
+    - user: 939
+    - group: 939
+    - template: jinja
+    - context:
+        repo_type: "yara"
+
+
+create_custom_local_sigma_repo_template:
+  git.present:
+    - name: /nsm/rules/custom-local-repos/local-sigma
+    - bare: False
+    - force: True
+
+add_readme_custom_local_sigma_repo_template:
+  file.managed:
+    - name: /nsm/rules/custom-local-repos/local-sigma/README
+    - source: salt://soc/files/soc/detections_custom_repo_template_readme.jinja
+    - user: 939
+    - group: 939
+    - template: jinja
+    - context:
+        repo_type: "sigma"
+
+socore_own_custom_repos:
+  file.directory:
+    - name: /nsm/rules/custom-local-repos/
+    - user: socore
+    - group: socore
+    - recurse:
+      - user
+      - group
+  
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/soc/defaults.yaml

@@ -1342,11 +1342,17 @@ soc:
                 license: Elastic-2.0
                 folder: sigma/stable
                 community: true
+              - repo: file:///nsm/rules/custom-local-repos/local-sigma
+                license: Elastic-2.0
+                community: false
             airgap:
               - repo: file:///nsm/rules/detect-sigma/repos/securityonion-resources
                 license: Elastic-2.0
                 folder: sigma/stable
                 community: true
+              - repo: file:///nsm/rules/custom-local-repos/local-sigma
+                license: Elastic-2.0
+                community: false
           sigmaRulePackages:
             - core
             - emerging_threats_addon
@@ -1412,10 +1418,16 @@ soc:
               - repo: https://github.com/Security-Onion-Solutions/securityonion-yara
                 license: DRL
                 community: true
+              - repo: file:///nsm/rules/custom-local-repos/local-yara
+                license: Elastic-2.0
+                community: false
             airgap:
               - repo: file:///nsm/rules/detect-yara/repos/securityonion-yara
                 license: DRL
                 community: true
+              - repo: file:///nsm/rules/custom-local-repos/local-yara
+                license: Elastic-2.0
+                community: false
           yaraRulesFolder: /opt/sensoroni/yara/rules
           stateFilePath: /opt/sensoroni/fingerprints/strelkaengine.state
           integrityCheckFrequencySeconds: 1200
@@ -1435,6 +1447,8 @@ soc:
           rulesFingerprintFile: /opt/sensoroni/fingerprints/emerging-all.fingerprint
           stateFilePath: /opt/sensoroni/fingerprints/suricataengine.state
           integrityCheckFrequencySeconds: 1200
+          ignoredSidRanges:
+            - '1100000-1101000'
       client:
         enableReverseLookup: false
         docsUrl: /docs/

salt/soc/files/soc/detections_custom_repo_template_readme.jinja

@@ -0,0 +1,94 @@
+{% if repo_type == 'yara' %}
+# YARA Local Custom Rules Repository
+
+This folder has already been initialized as a git repo 
+and your Security Onion grid is configured to import any YARA rule files found here. 
+
+Just add your rule file and commit it.
+
+For example:
+
+** Note: If this is your first time making changes to this repo, you may run into the following error:
+
+fatal: detected dubious ownership in repository at '/nsm/rules/custom-local-repos/local-yara'
+To add an exception for this directory, call:
+	git config --global --add safe.directory /nsm/rules/custom-local-repos/local-yara
+
+This means that the user you are running commands as does not match the user that is used for this git repo (socore).
+You will need to make sure your rule files are accessible to the socore user, so either su to socore 
+or add the exception and then chown the rule files later.
+
+Also, you will be asked to set some configuration:
+```
+Author identity unknown
+*** Please tell me who you are.
+Run
+  git config --global user.email "you@example.com"
+  git config --global user.name "Your Name"
+to set your account's default identity.
+Omit --global to set the identity only in this repository.
+```
+
+Run these commands, ommitting the `--global`.
+
+With that out of the way:
+
+First, create the rule file with a .yar extension:
+`vi my_custom_rule.yar`
+
+Next, use git to stage the new rule to be committed:
+`git add my_custom_rule.yar`
+
+Finally, commit it:
+`git commit -m "Initial commit of my_custom_rule.yar"`
+
+The next time the Strelka / YARA engine syncs, the new rule should be imported
+If there are errors, review the sync log to troubleshoot further.
+
+{% elif repo_type == 'sigma' %}
+# Sigma Local Custom Rules Repository
+
+This folder has already been initialized as a git repo 
+and your Security Onion grid is configured to import any Sigma rule files found here. 
+
+Just add your rule file and commit it.
+
+For example:
+
+** Note: If this is your first time making changes to this repo, you may run into the following error:
+
+fatal: detected dubious ownership in repository at '/nsm/rules/custom-local-repos/local-sigma'
+To add an exception for this directory, call:
+	git config --global --add safe.directory /nsm/rules/custom-local-repos/local-sigma
+
+This means that the user you are running commands as does not match the user that is used for this git repo (socore).
+You will need to make sure your rule files are accessible to the socore user, so either su to socore 
+or add the exception and then chown the rule files later.
+
+Also, you will be asked to set some configuration:
+```
+Author identity unknown
+*** Please tell me who you are.
+Run
+  git config --global user.email "you@example.com"
+  git config --global user.name "Your Name"
+to set your account's default identity.
+Omit --global to set the identity only in this repository.
+```
+
+Run these commands, ommitting the `--global`.
+
+With that out of the way:
+
+First, create the rule file with a .yml or .yaml extension:
+`vi my_custom_rule.yml`
+
+Next, use git to stage the new rule to be committed:
+`git add my_custom_rule.yml`
+
+Finally, commit it:
+`git commit -m "Initial commit of my_custom_rule.yml"`
+
+The next time the Elastalert / Sigma engine syncs, the new rule should be imported
+If there are errors, review the sync log to troubleshoot further.
+{% endif %}

salt/soc/soc_soc.yaml

@@ -390,6 +390,12 @@ soc:
             advanced: True
             forcedType: "[]{}"
             helpLink: suricata.html
+          ignoredSidRanges:
+            description: 'List of Suricata SID ranges to ignore during the Integrity Check. This is useful for ignoring specific rules not governed by the UI. Each line should contain 1 range in the format "1100000-1200000". The ranges are treated as inclusive.'
+            global: True
+            advanced: True
+            forcedType: "[]string"
+            helpLink: detections.html#rule-engine-status
       client:
         enableReverseLookup:
           description: Set to true to enable reverse DNS lookups for IP addresses in the SOC UI.

setup/so-functions

@@ -1878,9 +1878,9 @@ repo_sync_local() {
 			fi
 			dnf install -y yum-utils device-mapper-persistent-data lvm2
 			curl -fsSL https://repo.securityonion.net/file/so-repo/prod/2.4/so/so.repo | tee /etc/yum.repos.d/so.repo
-			rpm --import https://repo.saltproject.io/salt/py3/redhat/9/x86_64/SALT-PROJECT-GPG-PUBKEY-2023.pub
+			rpm --import https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public
 			dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
-			curl -fsSL "https://repo.saltproject.io/salt/py3/redhat/9/x86_64/minor/$SALTVERSION.repo" | tee /etc/yum.repos.d/salt.repo
+			curl -fsSL "https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" | tee /etc/yum.repos.d/salt.repo
 			dnf repolist
 			curl --retry 5 --retry-delay 60 -A "netinstall/$SOVERSION/$OS/$(uname -r)/1" https://sigs.securityonion.net/checkup --output /tmp/install
 		else
@@ -1913,27 +1913,22 @@ saltify() {
 		logCmd "mkdir -vp /etc/apt/keyrings"
 		logCmd "wget -q --inet4-only -O /etc/apt/keyrings/docker.pub https://download.docker.com/linux/ubuntu/gpg"
 
+		# Download public key
+		logCmd "curl -fsSL -o /etc/apt/keyrings/salt-archive-keyring-2023.pgp https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public"
+		# Create apt repo target configuration
+		echo "deb [signed-by=/etc/apt/keyrings/salt-archive-keyring-2023.pgp arch=amd64] https://packages.broadcom.com/artifactory/saltproject-deb/ stable main" | sudo tee /etc/apt/sources.list.d/salt.list
+
 		if [[ $is_ubuntu ]]; then
-
-			# Add Salt Repo
-			logCmd "curl -fsSL -o /etc/apt/keyrings/salt-archive-keyring-2023.gpg https://repo.saltproject.io/salt/py3/ubuntu/$UBVER/amd64/minor/$SALTVERSION/SALT-PROJECT-GPG-PUBKEY-2023.gpg"
-			echo "deb [signed-by=/etc/apt/keyrings/salt-archive-keyring-2023.gpg] https://repo.saltproject.io/salt/py3/ubuntu/$UBVER/amd64/minor/$SALTVERSION/ $OSVER main" | sudo tee /etc/apt/sources.list.d/salt.list
-
 			# Add Docker Repo
 			add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" 
 		
 		else
-			# Add Salt Repo *NOTE* You have to use debian 11 since it isn't out for 12
-			logCmd "curl -fsSL -o /etc/apt/keyrings/salt-archive-keyring-2023.gpg https://repo.saltproject.io/salt/py3/debian/11/amd64/minor/$SALTVERSION/SALT-PROJECT-GPG-PUBKEY-2023.gpg"
-			echo "deb [signed-by=/etc/apt/keyrings/salt-archive-keyring-2023.gpg] https://repo.saltproject.io/salt/py3/debian/11/amd64/minor/$SALTVERSION/ bullseye main" | sudo tee /etc/apt/sources.list.d/salt.list
-
 			# Add Docker Repo
 			curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
 			echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $OSVER stable" > /etc/apt/sources.list.d/docker.list 
-		
 		fi
 
-		logCmd "apt-key add /etc/apt/keyrings/salt-archive-keyring-2023.gpg"
+		logCmd "apt-key add /etc/apt/keyrings/salt-archive-keyring-2023.pgp"
 
 		#logCmd "apt-key add /opt/so/gpg/SALTSTACK-GPG-KEY.pub"
 		logCmd "apt-key add /etc/apt/keyrings/docker.pub"