Merge pull request #8571 from Security-Onion-Solutions/dev

2.3.150

HOTFIX

@@ -1 +0,0 @@
-20220719 20220812

README.md

@@ -1,6 +1,6 @@
-## Security Onion 2.3.140
+## Security Onion 2.3.150
 
-Security Onion 2.3.140 is here!
+Security Onion 2.3.150 is here!
 
 ## Screenshots
 

VERIFY_ISO.md

@@ -1,18 +1,18 @@
-### 2.3.140-20220812 ISO image built on 2022/08/12
+### 2.3.150-20220820 ISO image built on 2022/08/20
 
 
 
 ### Download and Verify
 
-2.3.140-20220812 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.3.140-20220812.iso
+2.3.150-20220820 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.3.150-20220820.iso
 
-MD5: 13D4A5D663B5A36D045B980E5F33E6BC  
-SHA1: 85DC36B7E96575259DFD080BC860F6508D5F5899  
-SHA256: DE5D0F82732B81456180AA40C124E5C82688611941EEAF03D85986806631588C 
+MD5: D2C0B67F19C18F0AB6FD1EC9B1E4034A  
+SHA1: F14BF42C6C634BDECA654B169FE6815BB6798F70  
+SHA256: 9E37E5CCCBD209486EB79E8F991DE83F64E2208D32E5B56F8E0A6C3933EB42AC 
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.140-20220812.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.150-20220820.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS  
@@ -26,22 +26,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/ma
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.140-20220812.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.150-20220820.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.3.140-20220812.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.3.150-20220820.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.3.140-20220812.iso.sig securityonion-2.3.140-20220812.iso
+gpg --verify securityonion-2.3.150-20220820.iso.sig securityonion-2.3.150-20220820.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Fri 12 Aug 2022 03:59:11 PM EDT using RSA key ID FE507013
+gpg: Signature made Sat 20 Aug 2022 08:07:10 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

VERSION

@@ -1 +1 @@
-2.3.140
+2.3.150

salt/common/tools/sbin/soup

@@ -203,7 +203,7 @@ check_airgap() {
 
 check_local_mods() {
   local salt_local=/opt/so/saltstack/local
-
+  local_ignore_arr=("/opt/so/saltstack/local/salt/zeek/policy/intel/intel.dat")
   local_mod_arr=()
 
   while IFS= read -r -d '' local_file; do
@@ -211,10 +211,12 @@ check_local_mods() {
     default_file="${DEFAULT_SALT_DIR}${stripped_path}"
     if [[ -f $default_file ]]; then
       file_diff=$(diff "$default_file" "$local_file" )
+      if [[ ! " ${local_ignore_arr[*]} " =~ " ${local_file} " ]]; then
         if [[ $(echo "$file_diff" | grep -c "^<") -gt 0 ]]; then
           local_mod_arr+=( "$local_file" )
         fi
       fi
+    fi
   done< <(find $salt_local -type f -print0)
 
   if [[ ${#local_mod_arr} -gt 0 ]]; then
@@ -223,11 +225,24 @@ check_local_mods() {
       echo "  $file_str"
     done
     echo ""
-    echo "To reference this list later, check $SOUP_LOG"
-    sleep 10
+    echo "To reference this list later, check $SOUP_LOG".
+    echo
+    if [[ -z $UNATTENDED ]] && ! [[ "${1}" == "skip-prompt" ]]; then
+      while true; do
+        read -p "Please review the local modifications shown above as they may cause problems during or after the update.
+
+Would you like to proceed with the update anyway?
+
+If so, type 'YES'. Otherwise, type anything else to exit SOUP. " yn
+
+        case $yn in
+          [yY][eE][sS] ) echo "Local modifications accepted.  Continuing..."; break;;
+          * ) exit 0;;
+        esac
+      done
+    fi
   fi
 }
-
 # {% endraw %}
 
 check_pillar_items() {
@@ -372,12 +387,7 @@ clone_to_tmp() {
 }
 
 elastalert_indices_check() {
-
-  # Stop Elastalert to prevent Elastalert indices from being re-created
-  if grep -q "^so-elastalert$" /opt/so/conf/so-status/so-status.conf ; then
-    so-elastalert-stop || true
-  fi
-  
+  echo "Checking Elastalert indices for compatibility..."
   # Wait for ElasticSearch to initialize
   echo -n "Waiting for ElasticSearch..."
   COUNT=0
@@ -403,6 +413,14 @@ elastalert_indices_check() {
     exit 1
   fi
 
+  MAJOR_ES_VERSION=$(so-elasticsearch-query / | jq -r .version.number | cut -d '.' -f1)
+  if [[ "$MAJOR_ES_VERSION" -lt "8" ]]; then
+
+    # Stop Elastalert to prevent Elastalert indices from being re-created
+    if grep -q "^so-elastalert$" /opt/so/conf/so-status/so-status.conf ; then
+      so-elastalert-stop || true
+    fi
+
     # Check Elastalert indices
     echo "Deleting Elastalert indices to prevent issues with upgrade to Elastic 8..."
     CHECK_COUNT=0
@@ -416,10 +434,9 @@ elastalert_indices_check() {
       COUNT=0
       ELASTALERT_INDICES_DELETED="no"
       while [[ "$COUNT" -le 240 ]]; do
-      RESPONSE=$(so-elasticsearch-query elastalert*)
+        RESPONSE=$(so-elasticsearch-query "elastalert*")
         if [[ "$RESPONSE" == "{}" ]]; then
           ELASTALERT_INDICES_DELETED="yes"
-        echo "Elastalert indices successfully deleted."
           break
         else
           ((COUNT+=1))
@@ -431,12 +448,17 @@ elastalert_indices_check() {
     done
 
     # If we were unable to delete the Elastalert indices, exit the script
-  if [ "$ELASTALERT_INDICES_DELETED" == "no" ]; then
+    if [ "$ELASTALERT_INDICES_DELETED" == "yes" ]; then
+      echo "Elastalert indices successfully deleted."
+    else
       echo
       echo -e "Unable to connect to delete Elastalert indices. Exiting."
       echo
       exit 1
     fi
+  else
+    echo "Major Elasticsearch version is 8 or greater...skipping Elastalert index maintenance."
+  fi
 }
 
 enable_highstate() {
@@ -448,7 +470,7 @@ enable_highstate() {
 es_version_check() {
     CHECK_ES=$(echo $INSTALLEDVERSION | awk -F. '{print $3}')
   
-    if [ "$CHECK_ES" -lt "110" ]; then
+    if [[ "$CHECK_ES" -lt "110" ]]; then
       echo "You are currently running Security Onion $INSTALLEDVERSION. You will need to update to version 2.3.130 before updating to 2.3.140 or higher."
       echo ""
       echo "If your deployment has Internet access, you can use the following command to update to 2.3.130:"
@@ -1218,6 +1240,7 @@ main() {
   set_palette
   check_elastic_license
   echo ""
+  check_local_mods
   check_os_updates
 
   echo "Generating new repo archive"
@@ -1382,7 +1405,7 @@ main() {
     fi
 
     echo "Checking for local modifications."
-    check_local_mods
+    check_local_mods skip-prompt
 
     echo "Checking sudoers file."
     check_sudoers

salt/elasticsearch/defaults.yaml

@@ -55,6 +55,10 @@ elasticsearch:
     indices:
       id_field_data:
         enabled: false
+    ingest:
+      geoip:
+        downloader:
+          enabled: false
     logger:
       org:
         elasticsearch:

salt/grafana/defaults.yaml

@@ -3085,12 +3085,6 @@ grafana:
             y: 16
             h: 8
             w: 24
-        elasticsearch_pipeline_time_nontc_graph:
-          gridPos:
-            x: 0
-            y: 24
-            h: 8
-            w: 24
 
 
     pipeline_overview_tc:
@@ -3140,9 +3134,3 @@ grafana:
             y: 16
             h: 8
             w: 24
-        elasticsearch_pipeline_time_tc_graph:
-          gridPos:
-            x: 0
-            y: 24
-            h: 8
-            w: 24

salt/idstools/etc/rulecat.conf

@@ -31,7 +31,7 @@
   {%- elif RULESET == 'ETPRO' %}
 --etpro={{ OINKCODE }}
   {%- elif RULESET == 'TALOS' %}
---url=https://www.snort.org/rules/snortrules-snapshot-2983.tar.gz?oinkcode={{ OINKCODE }}
+--url=https://www.snort.org/rules/snortrules-snapshot-29200.tar.gz?oinkcode={{ OINKCODE }}
   {%- endif %}
 {%- endif %}
 {%- if URLS != None %}

salt/kibana/bin/so-kibana-config-load

@@ -59,7 +59,7 @@ update() {
 
     IFS=$'\r\n' GLOBIGNORE='*' command eval  'LINES=($(cat $1))'
     for i in "${LINES[@]}"; do
-      RESPONSE=$({{ ELASTICCURL }} -X PUT "localhost:5601/api/saved_objects/config/8.3.2" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ")
+      RESPONSE=$({{ ELASTICCURL }} -X PUT "localhost:5601/api/saved_objects/config/8.3.3" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ")
       echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi
     done
  

salt/kibana/files/config_saved_objects.ndjson

@@ -1 +1 @@
-{"attributes": {"buildNum": 39457,"defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n  \"from\": \"now-24h\",\n  \"to\": \"now\"\n}"},"coreMigrationVersion": "8.3.2","id": "8.3.2","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="}
+{"attributes": {"buildNum": 39457,"defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n  \"from\": \"now-24h\",\n  \"to\": \"now\"\n}"},"coreMigrationVersion": "8.3.3","id": "8.3.3","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="}

salt/salt/minion.sls

@@ -81,11 +81,20 @@ set_log_levels:
       - "log_level: error"
       - "log_level_logfile: error"
 
-salt_minion_service_unit_file:
-  file.managed:
+delete_pre_150_start_delay:
+  file.line:
     - name: {{ SYSTEMD_UNIT_FILE }}
-    - source: salt://salt/service/salt-minion.service.jinja
+    - match: ^ExecStartPre=*
+    - mode: delete
+    - onchanges_in:
+      - module: systemd_reload
+
+salt_minion_service_start_delay:
+  file.managed:
+    - name: /etc/systemd/system/salt-minion.service.d/start-delay.conf
+    - source: salt://salt/service/start-delay.conf.jinja
     - template: jinja
+    - makedirs: True
     - defaults:
         service_start_delay: {{ service_start_delay }}
     - onchanges_in:
@@ -109,7 +118,7 @@ salt_minion_service:
       - file: mine_functions
 {% if INSTALLEDSALTVERSION|string == SALTVERSION|string %}
       - file: set_log_levels
-      - file: salt_minion_service_unit_file
+      - file: salt_minion_service_start_delay
 {% endif %}
     - order: last
 

salt/salt/service/salt-minion.service.jinja

@@ -1,15 +0,0 @@
-[Unit]
-Description=The Salt Minion
-Documentation=man:salt-minion(1) file:///usr/share/doc/salt/html/contents.html https://docs.saltstack.com/en/latest/contents.html
-After=network.target salt-master.service
-
-[Service]
-KillMode=process
-Type=notify
-NotifyAccess=all
-LimitNOFILE=8192
-ExecStart=/usr/bin/salt-minion
-ExecStartPre=/bin/sleep {{ salt['pillar.get']('salt:minion:service_start_delay', service_start_delay) }}
-
-[Install]
-WantedBy=multi-user.target

salt/salt/service/start-delay.conf.jinja

@@ -0,0 +1,2 @@
+[Service]
+ExecStartPre=/bin/sleep {{ salt['pillar.get']('salt:minion:service_start_delay', service_start_delay) }}

salt/soc/files/soc/menu.actions.json

@@ -19,7 +19,7 @@
               "/joblookup?esid={:soc_id}&time={:@timestamp}", 
               "/joblookup?ncid={:network.community_id}&time={:@timestamp}"
             ],
-            "categories": ["hunt", "alerts"]},
+            "categories": ["hunt", "alerts", "dashboards"]},
   { "name": "actionCyberChef", "description": "actionCyberChefHelp", "icon": "fas fa-bread-slice", "target": "_blank", 
             "links": [
               "/cyberchef/#input={value|base64}"

salt/soc/files/soc/presets.tlp.json

@@ -1,8 +1,9 @@
 {
   "labels": [
-    "white",
+    "clear",
     "green",
     "amber",
+    "amber+strict",
     "red"
   ],
   "customEnabled": false

salt/top.sls

@@ -84,7 +84,9 @@ base:
     {%- if STRELKA %}
     - strelka
     {%- endif %}
+    {%- if FILEBEAT %}
     - filebeat
+    {%- endif %}
     {%- if FLEETMANAGER or FLEETNODE %}
     - fleet.install_package
     {%- endif %}
@@ -433,7 +435,9 @@ base:
     - redis
     - fleet
     - fleet.install_package
+    {%- if FILEBEAT %}
     - filebeat
+    {%- endif %}
     - schedule
     - docker_clean
 
@@ -507,7 +511,9 @@ base:
     {%- endif %}
     - schedule
     - docker_clean
+    {%- if FILEBEAT %}
     - filebeat
+    {%- endif %}
     - idh
 
   'J@workstation:gui:enabled:^[Tt][Rr][Uu][Ee]$ and ( G@saltversion:{{saltversion}} and G@os:CentOS )':