Merge pull request #8571 from Security-Onion-Solutions/dev

2.3.150

HOTFIX

@@ -1 +0,0 @@
-20220719 20220812

README.md

@@ -1,6 +1,6 @@
-## Security Onion 2.3.140
+## Security Onion 2.3.150
 
-Security Onion 2.3.140 is here!
+Security Onion 2.3.150 is here!
 
 ## Screenshots
 

VERIFY_ISO.md

@@ -1,18 +1,18 @@
-### 2.3.140-20220812 ISO image built on 2022/08/12
+### 2.3.150-20220820 ISO image built on 2022/08/20
 
 
 
 ### Download and Verify
 
-2.3.140-20220812 ISO image:  
+2.3.150-20220820 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.3.140-20220812.iso
+https://download.securityonion.net/file/securityonion/securityonion-2.3.150-20220820.iso
 
-MD5: 13D4A5D663B5A36D045B980E5F33E6BC  
+MD5: D2C0B67F19C18F0AB6FD1EC9B1E4034A  
-SHA1: 85DC36B7E96575259DFD080BC860F6508D5F5899  
+SHA1: F14BF42C6C634BDECA654B169FE6815BB6798F70  
-SHA256: DE5D0F82732B81456180AA40C124E5C82688611941EEAF03D85986806631588C 
+SHA256: 9E37E5CCCBD209486EB79E8F991DE83F64E2208D32E5B56F8E0A6C3933EB42AC 
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.140-20220812.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.150-20220820.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS  
@@ -26,22 +26,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/ma
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.140-20220812.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.150-20220820.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.3.140-20220812.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.3.150-20220820.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.3.140-20220812.iso.sig securityonion-2.3.140-20220812.iso
+gpg --verify securityonion-2.3.150-20220820.iso.sig securityonion-2.3.150-20220820.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Fri 12 Aug 2022 03:59:11 PM EDT using RSA key ID FE507013
+gpg: Signature made Sat 20 Aug 2022 08:07:10 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

VERSION

@@ -1 +1 @@
-2.3.140
+2.3.150

salt/common/tools/sbin/soup

@@ -203,7 +203,7 @@ check_airgap() {
 
 check_local_mods() {
   local salt_local=/opt/so/saltstack/local
-
+  local_ignore_arr=("/opt/so/saltstack/local/salt/zeek/policy/intel/intel.dat")
   local_mod_arr=()
 
   while IFS= read -r -d '' local_file; do
@@ -211,8 +211,10 @@ check_local_mods() {
     default_file="${DEFAULT_SALT_DIR}${stripped_path}"
     if [[ -f $default_file ]]; then
       file_diff=$(diff "$default_file" "$local_file" )
-      if [[ $(echo "$file_diff" | grep -c "^<") -gt 0 ]]; then
+      if [[ ! " ${local_ignore_arr[*]} " =~ " ${local_file} " ]]; then
-        local_mod_arr+=( "$local_file" )
+        if [[ $(echo "$file_diff" | grep -c "^<") -gt 0 ]]; then
+          local_mod_arr+=( "$local_file" )
+        fi
       fi
     fi
   done< <(find $salt_local -type f -print0)
@@ -223,11 +225,24 @@ check_local_mods() {
       echo "  $file_str"
     done
     echo ""
-    echo "To reference this list later, check $SOUP_LOG"
+    echo "To reference this list later, check $SOUP_LOG".
-    sleep 10
+    echo
+    if [[ -z $UNATTENDED ]] && ! [[ "${1}" == "skip-prompt" ]]; then
+      while true; do
+        read -p "Please review the local modifications shown above as they may cause problems during or after the update.
+
+Would you like to proceed with the update anyway?
+
+If so, type 'YES'. Otherwise, type anything else to exit SOUP. " yn
+
+        case $yn in
+          [yY][eE][sS] ) echo "Local modifications accepted.  Continuing..."; break;;
+          * ) exit 0;;
+        esac
+      done
+    fi
   fi
 }
-
 # {% endraw %}
 
 check_pillar_items() {
@@ -372,12 +387,7 @@ clone_to_tmp() {
 }
 
 elastalert_indices_check() {
-
+  echo "Checking Elastalert indices for compatibility..."
-  # Stop Elastalert to prevent Elastalert indices from being re-created
-  if grep -q "^so-elastalert$" /opt/so/conf/so-status/so-status.conf ; then
-    so-elastalert-stop || true
-  fi
-  
   # Wait for ElasticSearch to initialize
   echo -n "Waiting for ElasticSearch..."
   COUNT=0
@@ -394,8 +404,8 @@ elastalert_indices_check() {
         echo -n "."
       fi
   done
-  
+
-  # Unable to connect to Elasticsearch 
+  # Unable to connect to Elasticsearch
   if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
     echo
     echo -e "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'sudo docker ps' \n  -running 'sudo so-elastic-restart'"
@@ -403,39 +413,51 @@ elastalert_indices_check() {
     exit 1
   fi
 
-  # Check Elastalert indices
+  MAJOR_ES_VERSION=$(so-elasticsearch-query / | jq -r .version.number | cut -d '.' -f1)
-  echo "Deleting Elastalert indices to prevent issues with upgrade to Elastic 8..."
+  if [[ "$MAJOR_ES_VERSION" -lt "8" ]]; then
-  CHECK_COUNT=0
+
-  while [[ "$CHECK_COUNT" -le 2 ]]; do
+    # Stop Elastalert to prevent Elastalert indices from being re-created
-    # Delete Elastalert indices
+    if grep -q "^so-elastalert$" /opt/so/conf/so-status/so-status.conf ; then
-    for i in $(so-elasticsearch-query _cat/indices | grep elastalert | awk '{print $3}'); do
+      so-elastalert-stop || true
-      so-elasticsearch-query $i -XDELETE;
+    fi
+
+    # Check Elastalert indices
+    echo "Deleting Elastalert indices to prevent issues with upgrade to Elastic 8..."
+    CHECK_COUNT=0
+    while [[ "$CHECK_COUNT" -le 2 ]]; do
+      # Delete Elastalert indices
+      for i in $(so-elasticsearch-query _cat/indices | grep elastalert | awk '{print $3}'); do
+        so-elasticsearch-query $i -XDELETE;
+         done
+
+      # Check to ensure Elastalert indices are deleted
+      COUNT=0
+      ELASTALERT_INDICES_DELETED="no"
+      while [[ "$COUNT" -le 240 ]]; do
+        RESPONSE=$(so-elasticsearch-query "elastalert*")
+        if [[ "$RESPONSE" == "{}" ]]; then
+          ELASTALERT_INDICES_DELETED="yes"
+          break
+        else
+          ((COUNT+=1))
+          sleep 1
+          echo -n "."
+        fi
+      done
+      ((CHECK_COUNT+=1))
     done
 
-    # Check to ensure Elastalert indices are deleted
+    # If we were unable to delete the Elastalert indices, exit the script
-    COUNT=0
+    if [ "$ELASTALERT_INDICES_DELETED" == "yes" ]; then
-    ELASTALERT_INDICES_DELETED="no"
+      echo "Elastalert indices successfully deleted."
-    while [[ "$COUNT" -le 240 ]]; do
+    else
-      RESPONSE=$(so-elasticsearch-query elastalert*)
+      echo
-      if [[ "$RESPONSE" == "{}" ]]; then
+      echo -e "Unable to connect to delete Elastalert indices. Exiting."
-        ELASTALERT_INDICES_DELETED="yes"
+      echo
-        echo "Elastalert indices successfully deleted."
+      exit 1
-        break
+    fi
-      else
+  else
-        ((COUNT+=1))
+    echo "Major Elasticsearch version is 8 or greater...skipping Elastalert index maintenance."
-        sleep 1
-        echo -n "."
-      fi
-    done
-    ((CHECK_COUNT+=1))
-  done
-
-  # If we were unable to delete the Elastalert indices, exit the script
-  if [ "$ELASTALERT_INDICES_DELETED" == "no" ]; then
-    echo
-    echo -e "Unable to connect to delete Elastalert indices. Exiting."
-    echo
-    exit 1
   fi
 }
 
@@ -448,7 +470,7 @@ enable_highstate() {
 es_version_check() {
     CHECK_ES=$(echo $INSTALLEDVERSION | awk -F. '{print $3}')
   
-    if [ "$CHECK_ES" -lt "110" ]; then
+    if [[ "$CHECK_ES" -lt "110" ]]; then
       echo "You are currently running Security Onion $INSTALLEDVERSION. You will need to update to version 2.3.130 before updating to 2.3.140 or higher."
       echo ""
       echo "If your deployment has Internet access, you can use the following command to update to 2.3.130:"
@@ -1218,6 +1240,7 @@ main() {
   set_palette
   check_elastic_license
   echo ""
+  check_local_mods
   check_os_updates
 
   echo "Generating new repo archive"
@@ -1382,7 +1405,7 @@ main() {
     fi
 
     echo "Checking for local modifications."
-    check_local_mods
+    check_local_mods skip-prompt
 
     echo "Checking sudoers file."
     check_sudoers

salt/elasticsearch/defaults.yaml

@@ -55,6 +55,10 @@ elasticsearch:
     indices:
       id_field_data:
         enabled: false
+    ingest:
+      geoip:
+        downloader:
+          enabled: false
     logger:
       org:
         elasticsearch:

salt/grafana/defaults.yaml

@@ -3085,12 +3085,6 @@ grafana:
             y: 16
             h: 8
             w: 24
-        elasticsearch_pipeline_time_nontc_graph:
-          gridPos:
-            x: 0
-            y: 24
-            h: 8
-            w: 24
 
 
     pipeline_overview_tc:
@@ -3140,9 +3134,3 @@ grafana:
             y: 16
             h: 8
             w: 24
-        elasticsearch_pipeline_time_tc_graph:
-          gridPos:
-            x: 0
-            y: 24
-            h: 8
-            w: 24

salt/idstools/etc/rulecat.conf

@@ -31,11 +31,11 @@
   {%- elif RULESET == 'ETPRO' %}
 --etpro={{ OINKCODE }}
   {%- elif RULESET == 'TALOS' %}
---url=https://www.snort.org/rules/snortrules-snapshot-2983.tar.gz?oinkcode={{ OINKCODE }}
+--url=https://www.snort.org/rules/snortrules-snapshot-29200.tar.gz?oinkcode={{ OINKCODE }}
   {%- endif %}
 {%- endif %}
 {%- if URLS != None %}
 {%- for URL in URLS %}
 --url={{ URL }}
 {%- endfor %}
-{%- endif %}
+{%- endif %}

salt/kibana/bin/so-kibana-config-load

@@ -59,7 +59,7 @@ update() {
 
     IFS=$'\r\n' GLOBIGNORE='*' command eval  'LINES=($(cat $1))'
     for i in "${LINES[@]}"; do
-      RESPONSE=$({{ ELASTICCURL }} -X PUT "localhost:5601/api/saved_objects/config/8.3.2" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ")
+      RESPONSE=$({{ ELASTICCURL }} -X PUT "localhost:5601/api/saved_objects/config/8.3.3" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ")
       echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi
     done
  

salt/kibana/files/config_saved_objects.ndjson

@@ -1 +1 @@
-{"attributes": {"buildNum": 39457,"defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n  \"from\": \"now-24h\",\n  \"to\": \"now\"\n}"},"coreMigrationVersion": "8.3.2","id": "8.3.2","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="}
+{"attributes": {"buildNum": 39457,"defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n  \"from\": \"now-24h\",\n  \"to\": \"now\"\n}"},"coreMigrationVersion": "8.3.3","id": "8.3.3","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="}

salt/salt/minion.sls

@@ -81,11 +81,20 @@ set_log_levels:
       - "log_level: error"
       - "log_level_logfile: error"
 
-salt_minion_service_unit_file:
+delete_pre_150_start_delay:
-  file.managed:
+  file.line:
     - name: {{ SYSTEMD_UNIT_FILE }}
-    - source: salt://salt/service/salt-minion.service.jinja
+    - match: ^ExecStartPre=*
+    - mode: delete
+    - onchanges_in:
+      - module: systemd_reload
+
+salt_minion_service_start_delay:
+  file.managed:
+    - name: /etc/systemd/system/salt-minion.service.d/start-delay.conf
+    - source: salt://salt/service/start-delay.conf.jinja
     - template: jinja
+    - makedirs: True
     - defaults:
         service_start_delay: {{ service_start_delay }}
     - onchanges_in:
@@ -109,7 +118,7 @@ salt_minion_service:
       - file: mine_functions
 {% if INSTALLEDSALTVERSION|string == SALTVERSION|string %}
       - file: set_log_levels
-      - file: salt_minion_service_unit_file
+      - file: salt_minion_service_start_delay
 {% endif %}
     - order: last
 

salt/salt/service/salt-minion.service.jinja

@@ -1,15 +0,0 @@
-[Unit]
-Description=The Salt Minion
-Documentation=man:salt-minion(1) file:///usr/share/doc/salt/html/contents.html https://docs.saltstack.com/en/latest/contents.html
-After=network.target salt-master.service
-
-[Service]
-KillMode=process
-Type=notify
-NotifyAccess=all
-LimitNOFILE=8192
-ExecStart=/usr/bin/salt-minion
-ExecStartPre=/bin/sleep {{ salt['pillar.get']('salt:minion:service_start_delay', service_start_delay) }}
-
-[Install]
-WantedBy=multi-user.target

salt/salt/service/start-delay.conf.jinja

@@ -0,0 +1,2 @@
+[Service]
+ExecStartPre=/bin/sleep {{ salt['pillar.get']('salt:minion:service_start_delay', service_start_delay) }}

salt/soc/files/soc/menu.actions.json

@@ -19,7 +19,7 @@
               "/joblookup?esid={:soc_id}&time={:@timestamp}", 
               "/joblookup?ncid={:network.community_id}&time={:@timestamp}"
             ],
-            "categories": ["hunt", "alerts"]},
+            "categories": ["hunt", "alerts", "dashboards"]},
   { "name": "actionCyberChef", "description": "actionCyberChefHelp", "icon": "fas fa-bread-slice", "target": "_blank", 
             "links": [
               "/cyberchef/#input={value|base64}"

salt/soc/files/soc/presets.pap.json

@@ -6,4 +6,4 @@
     "red"
   ],
   "customEnabled": false
-}
+}

salt/soc/files/soc/presets.tlp.json

@@ -1,9 +1,10 @@
 {
   "labels": [
-    "white",
+    "clear",
     "green",
     "amber",
+    "amber+strict",
     "red"
   ],
   "customEnabled": false
-}
+}

salt/top.sls

@@ -84,7 +84,9 @@ base:
     {%- if STRELKA %}
     - strelka
     {%- endif %}
+    {%- if FILEBEAT %}
     - filebeat
+    {%- endif %}
     {%- if FLEETMANAGER or FLEETNODE %}
     - fleet.install_package
     {%- endif %}
@@ -433,7 +435,9 @@ base:
     - redis
     - fleet
     - fleet.install_package
+    {%- if FILEBEAT %}
     - filebeat
+    {%- endif %}
     - schedule
     - docker_clean
 
@@ -507,7 +511,9 @@ base:
     {%- endif %}
     - schedule
     - docker_clean
+    {%- if FILEBEAT %}
     - filebeat
+    {%- endif %}
     - idh
 
   'J@workstation:gui:enabled:^[Tt][Rr][Uu][Ee]$ and ( G@saltversion:{{saltversion}} and G@os:CentOS )':