Merge pull request #11374 from Security-Onion-Solutions/dev

2.3.270

.github/DISCUSSION_TEMPLATE/2-4.yml

@@ -0,0 +1,210 @@
+body:
+  - type: markdown
+    attributes:
+      value: >
+        ⚠️ This category is solely for conversations related to Security Onion 2.4 ⚠️
+  - type: dropdown
+    attributes:
+      label: Version
+      description: Which version of Security Onion 2.4.x are you asking about?
+      options:
+        -
+        - 2.4.0
+        - 2.4.1
+        - 2.4.2
+        - 2.4.3
+        - 2.4.4
+        - 2.4.5
+        - 2.4.10
+        - 2.4.20
+        - 2.4.30
+        - 2.4.40
+        - 2.4.50
+        - 2.4.60
+        - 2.4.70
+        - 2.4.80
+        - 2.4.90
+        - 2.4.100
+        - Other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Installation Method
+      description: How did you install Security Onion?
+      options:
+        -
+        - Security Onion ISO image
+        - Network installation on Red Hat derivative like Oracle, Rocky, Alma, etc.
+        - Network installation on Ubuntu
+        - Network installation on Debian
+        - Other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Description
+      description: >
+        Is this discussion about installation, configuration, upgrading, or other?
+      options:
+        -
+        - installation
+        - configuration
+        - upgrading
+        - other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Installation Type
+      description: >
+        When you installed, did you choose Import, Eval, Standalone, Distributed, or something else?
+      options:
+        -
+        - Import
+        - Eval
+        - Standalone
+        - Distributed
+        - other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Location
+      description: >
+        Is this deployment in the cloud, on-prem with Internet access, or airgap?
+      options:
+        -
+        - cloud
+        - on-prem with Internet access
+        - airgap
+        - other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Hardware Specs
+      description: >
+        Does your hardware meet or exceed the minimum requirements for your installation type as shown at https://docs.securityonion.net/en/2.4/hardware.html?
+      options:
+        -
+        - Meets minimum requirements
+        - Exceeds minimum requirements
+        - Does not meet minimum requirements
+        - other (please provide detail below)
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: CPU
+      description: How many CPU cores do you have?
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: RAM
+      description: How much RAM do you have?
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: Storage for /
+      description: How much storage do you have for the / partition?
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: Storage for /nsm
+      description: How much storage do you have for the /nsm partition?
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Network Traffic Collection
+      description: >
+        Are you collecting network traffic from a tap or span port?
+      options:
+        -
+        - tap
+        - span port
+        - other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Network Traffic Speeds
+      description: >
+        How much network traffic are you monitoring?
+      options:
+        -
+        - Less than 1Gbps
+        - 1Gbps to 10Gbps
+        - more than 10Gbps
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Status
+      description: >
+        Does SOC Grid show all services on all nodes as running OK?
+      options:
+        -
+        - Yes, all services on all nodes are running OK
+        - No, one or more services are failed (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Salt Status
+      description: >
+        Do you get any failures when you run "sudo salt-call state.highstate"?
+      options:
+        -
+        - Yes, there are salt failures (please provide detail below)
+        - No, there are no failures
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Logs
+      description: >
+        Are there any additional clues in /opt/so/log/?
+      options:
+        -
+        - Yes, there are additional clues in /opt/so/log/ (please provide detail below)
+        - No, there are no additional clues
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Detail
+      description: Please read the placeholder and then provide detailed information to help us help you.
+      placeholder: >-
+        STOP! Please read these guidelines in their entirety before typing!
+        
+        Community Support is considered best effort and there are no guarantees and no SLAs. If you need private, priority, or enterprise support, please consider purchasing support from Security Onion Solutions.
+        
+        Please review the Github Community Guidelines (see link on the right side of the page).
+        
+        Please be patient, courteous, and respectful. Disrespectful messages can result in being banned.
+        
+        Before posting for help, check the Help, FAQ, and other sections of the documentation (https://docs.securityonion.net/) to see if your question has already been answered there.
+        
+        Please do not tag an individual in a discussion unless that individual has already volunteered to help you in that discussion.
+        
+        When creating your discussion, please put a relevant and descriptive title in the Title field and avoid generic titles like Help. When copying text from your Security Onion deployment to the discussion, please copy as plain text when possible rather than taking a screenshot of the text. This allows others to search for and find your text.
+        
+        Avoid typing in ALL CAPS as this looks like YELLING!
+        
+        If you need to include a large section of output, please do so as an attached file or Github gist rather than including the output directly in the reply itself. 
+        
+        If you attach files, please make sure they are plain text format. No Word docs or PDFs please.
+
+    validations:
+      required: true
+  - type: checkboxes
+    attributes:
+      label: Guidelines
+      options:
+        - label: I have read the above statement and can confirm my post is relevant to Security Onion 2.4.
+          required: true

HOTFIX

@@ -1 +1 @@
-20230301
+

VERIFY_ISO.md

@@ -1,18 +1,18 @@
-### 2.3.220-20230301 ISO image built on 2023/03/01
+### 2.3.270-20231006 ISO image built on 2023/10/06
 
 
 
 ### Download and Verify
 
-2.3.220-20230301 ISO image:  
+2.3.270-20231006 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.3.220-20230301.iso
+https://download.securityonion.net/file/securityonion/securityonion-2.3.270-20231006.iso
 
-MD5: 76870CF09FF27893574FC104F9AC6642  
+MD5: 3FC7A37EA402A5F0C6609D7431387575  
-SHA1: CBF5B407C5982CA40C7660FE5CD9E3C6C551D280  
+SHA1: 979851603E431EE9670A1576E5DCCD838CEDA294  
-SHA256: 0719D441DF8B77266CE16F5FA182BF0680567BE7AD0AE36979D4FE8E0953F094 
+SHA256: 34F72EDEA9A62E1545347A31DEDEDD099D824466EC52B8674ACC7DB6D7E8B943 
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.220-20230301.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.270-20231006.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS  
@@ -26,22 +26,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/ma
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.220-20230301.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.270-20231006.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.3.220-20230301.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.3.270-20231006.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.3.220-20230301.iso.sig securityonion-2.3.220-20230301.iso
+gpg --verify securityonion-2.3.270-20231006.iso.sig securityonion-2.3.270-20231006.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Wed 01 Mar 2023 03:50:25 PM EST using RSA key ID FE507013
+gpg: Signature made Thu 21 Sep 2023 10:43:13 AM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

VERSION

@@ -1 +1 @@
-2.3.220
+2.3.270

pillar/zeek/init.sls

@@ -15,6 +15,7 @@ zeek:
     SpoolDir: /nsm/zeek/spool
     CfgDir: /opt/zeek/etc
     CompressLogs: 1
+    ZeekPort: 27760
   local:
     '@load':
       - misc/loaded-scripts

salt/common/tools/sbin/soup

@@ -17,9 +17,30 @@
 
 . /usr/sbin/so-common
 
+INSTALLEDVERSION=$(cat /etc/soversion)
+
+if [[ $INSTALLEDVERSION == "2.4.4" ]]; then
+
+  echo "Initiating supersoup mode"
+  mkdir -p /tmp/supersoup
+  cd /tmp/supersoup
+  echo "Updating soup..."
+  wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/salt/manager/tools/sbin/soup
+  cp soup /opt/so/saltstack/default/salt/manager/tools/sbin
+  echo "Updating soup..."
+  salt-call state.apply manager
+  echo "Please run soup a second time."
+
+  exit 0
+fi
+
+if [ "$INSTALLEDVERSION" = '2.4.3' ] || [ "$INSTALLEDVERSION" = '2.4.2' ] || [ "$INSTALLEDVERSION" = '2.4.1' ] || [ "$INSTALLEDVERSION" = '2.4.0' ]; then
+  echo "soup is not supported on $INSTALLEDVERSION. Please install the latest 2.4 release."
+  exit 1
+fi
+
 UPDATE_DIR=/tmp/sogh/securityonion
 DEFAULT_SALT_DIR=/opt/so/saltstack/default
-INSTALLEDVERSION=$(cat /etc/soversion)
 POSTVERSION=$INSTALLEDVERSION
 INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}')
 BATCHSIZE=5
@@ -212,7 +233,7 @@ check_local_mods() {
     if [[ -f $default_file ]]; then
       file_diff=$(diff "$default_file" "$local_file" )
       if [[ ! " ${local_ignore_arr[*]} " =~ " ${local_file} " ]]; then
-        if [[ $(echo "$file_diff" | grep -c "^<") -gt 0 ]]; then
+        if [[ $(echo "$file_diff" | grep -Ec "^[<>]") -gt 0 ]]; then
           local_mod_arr+=( "$local_file" )
         fi
       fi
@@ -554,6 +575,12 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == 2.3.190 ]] && up_to_2.3.200
     [[ "$INSTALLEDVERSION" == 2.3.200 ]] && up_to_2.3.210
     [[ "$INSTALLEDVERSION" == 2.3.210 ]] && up_to_2.3.220
+    [[ "$INSTALLEDVERSION" == 2.3.220 ]] && up_to_2.3.230
+    [[ "$INSTALLEDVERSION" == 2.3.230 ]] && up_to_2.3.240
+    [[ "$INSTALLEDVERSION" == 2.3.240 ]] && up_to_2.3.250
+    [[ "$INSTALLEDVERSION" == 2.3.250 ]] && up_to_2.3.260
+    [[ "$INSTALLEDVERSION" == 2.3.260 ]] && up_to_2.3.270
+    
     true
 }
 
@@ -580,6 +607,11 @@ postupgrade_changes() {
     [[ "$POSTVERSION" == 2.3.190 ]] && post_to_2.3.200
     [[ "$POSTVERSION" == 2.3.200 ]] && post_to_2.3.210
     [[ "$POSTVERSION" == 2.3.210 ]] && post_to_2.3.220
+    [[ "$POSTVERSION" == 2.3.220 ]] && post_to_2.3.230
+    [[ "$POSTVERSION" == 2.3.230 ]] && post_to_2.3.240
+    [[ "$POSTVERSION" == 2.3.240 ]] && post_to_2.3.250
+    [[ "$POSTVERSION" == 2.3.250 ]] && post_to_2.3.260
+    [[ "$POSTVERSION" == 2.3.260 ]] && post_to_2.3.270
 
     true
 }
@@ -713,6 +745,33 @@ post_to_2.3.220() {
   POSTVERSION=2.3.220
 }
 
+post_to_2.3.230() {
+  echo "Nothing to do for .230"
+  POSTVERSION=2.3.230
+}
+
+post_to_2.3.240() {
+  echo "Nothing to do for .240"
+  POSTVERSION=2.3.240
+}
+
+post_to_2.3.250() {
+  echo "Nothing to do for .250"
+  POSTVERSION=2.3.250
+}
+
+post_to_2.3.260() {
+  echo "Nothing to do for .260"
+  POSTVERSION=2.3.260
+}
+
+post_to_2.3.270() {
+  echo "Pruning unused docker volumes on all nodes - This process will run in the background."
+  salt --async \* cmd.run "docker volume prune -f"
+
+  POSTVERSION=2.3.270
+}
+
 stop_salt_master() {
     # kill all salt jobs across the grid because the hang indefinitely if they are queued and salt-master restarts
     set +e
@@ -1053,6 +1112,31 @@ up_to_2.3.220() {
   INSTALLEDVERSION=2.3.220
 }
 
+up_to_2.3.230() {
+  echo "Upgrading to 2.3.230"
+  INSTALLEDVERSION=2.3.230
+}
+
+up_to_2.3.240() {
+  echo "Upgrading to 2.3.240"
+  INSTALLEDVERSION=2.3.240
+}
+
+up_to_2.3.250() {
+  echo "Upgrading to 2.3.250"
+  INSTALLEDVERSION=2.3.250
+}
+
+up_to_2.3.260() {
+  echo "Upgrading to 2.3.260"
+  INSTALLEDVERSION=2.3.260
+}
+
+up_to_2.3.270() {
+  echo "Upgrading to 2.3.270"
+  INSTALLEDVERSION=2.3.270
+}
+
 verify_upgradespace() {
   CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//')
   if [ "$CURRENTSPACE" -lt "10" ]; then

salt/elasticsearch/files/ingest/suricata.dns

@@ -1,21 +1,21 @@
 {
   "description" : "suricata.dns",
   "processors" : [
-    { "rename": 	{ "field": "message2.proto", 		"target_field": "network.transport",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.proto", 			"target_field": "network.transport",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.app_proto", 	"target_field": "network.protocol",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.app_proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.type", 	"target_field": "dns.query.type",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dns.type", 		"target_field": "dns.query.type",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.tx_id",	"target_field": "dns.id",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dns.tx_id",		"target_field": "dns.id",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.version", 	"target_field": "dns.version",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dns.version", 		"target_field": "dns.version",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.rrname", 	"target_field": "dns.query.name",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dns.rrname", 		"target_field": "dns.query.name",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.rrtype", 	"target_field": "dns.query.type_name",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dns.rrtype", 		"target_field": "dns.query.type_name",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.flags", 	"target_field": "dns.flags",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dns.flags", 		"target_field": "dns.flags",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.qr", 		"target_field": "dns.qr",			"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dns.qr", 			"target_field": "dns.qr",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.rd", 		"target_field": "dns.recursion.desired",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dns.rd", 			"target_field": "dns.recursion.desired",	"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.ra", 		"target_field": "dns.recursion.available",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dns.ra", 			"target_field": "dns.recursion.available",	"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.dns.rcode", 	"target_field": "dns.response.code_name",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.dns.rcode", 		"target_field": "dns.response.code_name",	"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.grouped.A", 	"target_field": "dns.answers.data",		"ignore_missing": true 	} },
+    { "rename":		{ "field": "message2.dns.grouped.A", 		"target_field": "dns.answers.data",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.grouped.CNAME", 	"target_field": "dns.answers.name",		"ignore_missing": true 	} },
+    { "rename":		{ "field": "message2.dns.grouped.CNAME",	"target_field": "dns.answers.name",		"ignore_missing": true 	} },
-    { "pipeline": 	{ "if": "ctx.dns.query?.name != null && ctx.dns.query.name.contains('.')", "name": "dns.tld"			} },
+    { "pipeline":	{ "if": "ctx.dns.query?.name != null && ctx.dns.query.name.contains('.')", "name": "dns.tld"				} },
-    { "pipeline": { "name": "common" } }
+    { "pipeline": 	{ "name": "common"													} }
   ]
 }

salt/kibana/bin/so-kibana-config-load

@@ -59,7 +59,7 @@ update() {
 
     IFS=$'\r\n' GLOBIGNORE='*' command eval  'LINES=($(cat $1))'
     for i in "${LINES[@]}"; do
-      RESPONSE=$({{ ELASTICCURL }} -X PUT "localhost:5601/api/saved_objects/config/8.6.2" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ")
+      RESPONSE=$({{ ELASTICCURL }} -X PUT "localhost:5601/api/saved_objects/config/8.8.2" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ")
       echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi
     done
  

salt/kibana/files/config_saved_objects.ndjson

@@ -1 +1 @@
-{"attributes": {"buildNum": 39457,"defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n  \"from\": \"now-24h\",\n  \"to\": \"now\"\n}"},"coreMigrationVersion": "8.6.2","id": "8.6.2","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="}
+{"attributes": {"buildNum": 39457,"defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n  \"from\": \"now-24h\",\n  \"to\": \"now\"\n}"},"coreMigrationVersion": "8.8.2","id": "8.8.2","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="}

salt/playbook/init.sls

@@ -84,6 +84,14 @@ playbook_password_none:
 
 {% else %}
 
+playbookfilesdir:
+  file.directory:
+    - name: /opt/so/conf/playbook/redmine-files
+    - dir_mode: 775
+    - user: 939
+    - group: 939
+    - makedirs: True
+
 so-playbook:
   docker_container.running:
     - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-playbook:{{ VERSION }}
@@ -91,6 +99,7 @@ so-playbook:
     - name: so-playbook
     - binds:
       - /opt/so/log/playbook:/playbook/log:rw
+      - /opt/so/conf/playbook/redmine-files:/usr/src/redmine/files:rw
     - environment:
       - REDMINE_DB_MYSQL={{ MANAGERIP }}
       - REDMINE_DB_DATABASE=playbook

salt/redis/init.sls

@@ -52,6 +52,13 @@ redisconf:
     - group: 939
     - template: jinja
 
+redisdatadir:
+  file.directory:
+    - name: /nsm/redis/data
+    - user: 939
+    - group: 939
+    - makedirs: True
+
 so-redis:
   docker_container.running:
     - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-redis:{{ VERSION }}
@@ -64,6 +71,7 @@ so-redis:
       - /opt/so/log/redis:/var/log/redis:rw
       - /opt/so/conf/redis/etc/redis.conf:/usr/local/etc/redis/redis.conf:ro
       - /opt/so/conf/redis/working:/redis:rw
+      - /nsm/redis/data:/data:rw
       - /etc/pki/redis.crt:/certs/redis.crt:ro
       - /etc/pki/redis.key:/certs/redis.key:ro
   {% if grains['role'] in ['so-manager', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import'] %}

salt/sensoroni/files/analyzers/emailrep/requirements.txt

@@ -1,2 +1,2 @@
-requests>=2.27.1
+requests>=2.31.0
 pyyaml>=6.0

salt/sensoroni/files/analyzers/greynoise/requirements.txt

@@ -1,2 +1,2 @@
-requests>=2.27.1
+requests>=2.31.0
 pyyaml>=6.0

salt/sensoroni/files/analyzers/localfile/requirements.txt

@@ -1,2 +1,2 @@
-requests>=2.27.1
+requests>=2.31.0
 pyyaml>=6.0

salt/sensoroni/files/analyzers/malwarehashregistry/requirements.txt

@@ -1,2 +1,2 @@
-requests>=2.27.1
+requests>=2.31.0
 python-whois>=0.7.3

salt/sensoroni/files/analyzers/otx/requirements.txt

@@ -1,2 +1,2 @@
-requests>=2.27.1
+requests>=2.31.0
 pyyaml>=6.0

salt/sensoroni/files/analyzers/pulsedive/requirements.txt

@@ -1,2 +1,2 @@
-requests>=2.27.1
+requests>=2.31.0
 pyyaml>=6.0

salt/sensoroni/files/analyzers/urlhaus/requirements.txt

@@ -1,2 +1,2 @@
-requests>=2.27.1
+requests>=2.31.0
 pyyaml>=6.0