Merge branch 'dev' into kilo

salt/common/tools/sbin/soup

@@ -93,8 +93,7 @@ check_err() {
     fi
     set +e
     systemctl_func "start" "$cron_service_name"
-    echo "Ensuring highstate is enabled."
-    salt-call state.enable highstate --local
+    enable_highstate
     exit $exit_code
   fi
 
@@ -366,6 +365,12 @@ clone_to_tmp() {
   fi
 }
 
+enable_highstate() {
+    echo "Enabling highstate."
+    salt-call state.enable highstate -l info --local
+    echo ""
+}
+
 generate_and_clean_tarballs() {
   local new_version
   new_version=$(cat $UPDATE_DIR/VERSION)
@@ -492,10 +497,10 @@ stop_salt_master() {
     set +e
     echo ""
     echo "Killing all Salt jobs across the grid."
-    salt \* saltutil.kill_all_jobs
+    salt \* saltutil.kill_all_jobs >> $SOUP_LOG 2>&1
     echo ""
     echo "Killing any queued Salt jobs on the manager."
-    pkill -9 -ef "/usr/bin/python3 /bin/salt"
+    pkill -9 -ef "/usr/bin/python3 /bin/salt" >> $SOUP_LOG 2>&1
     set -e
 
     echo ""
@@ -857,7 +862,7 @@ upgrade_salt() {
     echo ""
     set +e
     run_check_net_err \
-    "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -F -M -x python3 stable \"$NEWSALTVERSION\"" \
+    "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \
     "Could not update salt, please check $SOUP_LOG for details."
     set -e
     echo "Applying apt hold for Salt."
@@ -866,11 +871,27 @@ upgrade_salt() {
     apt-mark hold "salt-master"
     apt-mark hold "salt-minion"
   fi
+
+  echo "Checking if Salt was upgraded."
+  echo ""
+  # Check that Salt was upgraded
+  SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk '{print $2}')
+  if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then
+    echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG."
+    echo "Once the issue is resolved, run soup again."
+    echo "Exiting."
+    echo ""
+    exit 0
+  else
+    echo "Salt upgrade success."
+    echo ""
+  fi
+
 }
 
 update_repo() {
-  echo "Performing repo changes."
   if [[ "$OS" == "centos" ]]; then
+    echo "Performing repo changes."
     # Import GPG Keys
     gpg_rpm_import
     echo "Disabling fastestmirror."
@@ -890,6 +911,21 @@ update_repo() {
       yum clean all
       yum repolist
     fi
+  elif [[ "$OS" == "ubuntu" ]]; then
+    ubuntu_version=$(grep VERSION_ID /etc/os-release | awk -F '[ "]' '{print $2}')
+
+    if grep -q "UBUNTU_CODENAME=bionic" /etc/os-release; then
+      OSVER=bionic
+    elif grep -q "UBUNTU_CODENAME=focal" /etc/os-release; then
+      OSVER=focal
+    else
+      echo "We do not support your current version of Ubuntu."
+      exit 1
+    fi
+
+    rm -f /etc/apt/sources.list.d/salt.list
+    echo "deb https://repo.securityonion.net/file/securityonion-repo/ubuntu/$ubuntu_version/amd64/salt $OSVER main" > /etc/apt/sources.list.d/saltstack.list
+    apt-get update
   fi
 }
 
@@ -922,6 +958,8 @@ verify_latest_update_script() {
 apply_hotfix() {
   if [[ "$INSTALLEDVERSION" == "2.3.90" ]] ; then
     fix_wazuh
+  elif [[ "$INSTALLEDVERSION" == "2.3.110" ]] ; then
+    2_3_10_hotfix_1
   else
     echo "No actions required. ($INSTALLEDVERSION/$HOTFIXVERSION)"
   fi
@@ -943,6 +981,28 @@ fix_wazuh() {
   fi
 }
 
+#upgrade salt to 3004.1
+2_3_10_hotfix_1() {
+    systemctl_func "stop" "$cron_service_name"
+    # update mine items prior to stopping salt-minion and salt-master
+    update_salt_mine
+    stop_salt_minion
+    stop_salt_master
+    update_repo
+    # Does salt need upgraded. If so update it.
+    if [[ $UPGRADESALT -eq 1 ]]; then
+      echo "Upgrading Salt"
+      # Update the repo files so it can actually upgrade
+      upgrade_salt
+    fi
+    rm -f /opt/so/state/influxdb_continuous_query.py.patched /opt/so/state/influxdbmod.py.patched /opt/so/state/influxdb_retention_policy.py.patched
+    systemctl_func "start" "salt-master"
+    salt-call state.apply salt.python3-influxdb -l info
+    systemctl_func "start" "salt-minion"
+    systemctl_func "start" "$cron_service_name"
+
+}
+
 main() {
   trap 'check_err $?' EXIT
   
@@ -1012,12 +1072,19 @@ main() {
   upgrade_check_salt
   set -e
 
+  if [[ $is_airgap -eq 0 ]]; then
+    update_centos_repo
+    yum clean all
+    check_os_updates
+  fi
+
   if [ "$is_hotfix" == "true" ]; then
     echo "Applying $HOTFIXVERSION hotfix"
     copy_new_files
     apply_hotfix
     echo "Hotfix applied"
     update_version
+    enable_highstate
     salt-call state.highstate -l info queue=True
   else
     echo ""
@@ -1032,9 +1099,6 @@ main() {
     echo "Updating dockers to $NEWVERSION."
     if [[ $is_airgap -eq 0 ]]; then
       airgap_update_dockers
-      update_centos_repo
-      yum clean all
-      check_os_updates
     # if not airgap but -f was used
     elif [[ ! -z "$ISOLOC" ]]; then
       airgap_update_dockers
@@ -1057,21 +1121,6 @@ main() {
       echo "Upgrading Salt"
       # Update the repo files so it can actually upgrade
       upgrade_salt
-    
-      echo "Checking if Salt was upgraded."
-      echo ""
-      # Check that Salt was upgraded
-      SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk '{print $2}')
-      if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then
-        echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG."
-        echo "Once the issue is resolved, run soup again."
-        echo "Exiting."
-        echo ""
-        exit 0
-      else
-        echo "Salt upgrade success."
-        echo ""
-      fi
     fi
 
     preupgrade_changes
@@ -1127,9 +1176,7 @@ main() {
       echo ""
     fi
 
-    echo "Enabling highstate."
-    salt-call state.enable highstate -l info --local
-    echo ""
+    enable_highstate
 
     echo ""
     echo "Running a highstate. This could take several minutes."

salt/idh/init.sls

@@ -19,11 +19,37 @@
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set MANAGER = salt['grains.get']('master') %}
+{% set MAININT = salt['pillar.get']('host:mainint') %}
+{% set MAINIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %}
+{% set RESTRICTIDHSERVICES = salt['pillar.get']('idh:restrict_management_ip', False) %}
 
 include:
   - idh.openssh.config
+  - firewall
 
-# IDH State
+
+# If True, block IDH Services from accepting connections on Managment IP
+{% if RESTRICTIDHSERVICES %}
+  {% from 'idh/opencanary_config.map.jinja' import OPENCANARYCONFIG %}
+  {% set idh_services = salt['pillar.get']('idh:services', []) %}
+
+  {% for service in idh_services %}
+  {% if service in ["smnp","ntp", "tftp"] %}
+    {% set proto = 'udp' %}
+  {% else %}
+    {% set proto = 'tcp' %}
+  {% endif %}
+block_mgt_ip_idh_services_{{ proto }}_{{ OPENCANARYCONFIG[service~'.port'] }} :
+  iptables.insert:
+    - table: filter
+    - chain: INPUT
+    - jump: DROP
+    - position: 1
+    - proto:  {{ proto }}
+    - dport: {{ OPENCANARYCONFIG[service~'.port'] }}
+    - destination: {{ MAINIP }}
+  {% endfor %}
+{% endif %}
 
 # Create a config directory
 temp:

salt/influxdb/defaults.yaml

@@ -1,4 +1,4 @@
-{% set measurements = salt['cmd.shell']('docker exec -t so-influxdb influx -format json -ssl -unsafeSsl -database telegraf -execute "show measurements" 2> /root/measurement_query.log | jq -r .results[0].series[0].values[]?[0] 2>> /root/measurement_query.log') %}
+{% set measurements = salt['cmd.shell']('docker exec -t so-influxdb influx -format json -ssl -unsafeSsl -database telegraf -execute "show measurements" 2> /root/measurement_query.log | jq -r .results[0].series[0].values[]?[0] 2>> /root/measurement_query.log', shell='/bin/bash') %}
 
 influxdb:
   retention_policies:

salt/repo/client/centos.sls

@@ -0,0 +1,98 @@
+{% from 'repo/client/map.jinja' import ABSENTFILES with context %}
+{% from 'repo/client/map.jinja' import REPOPATH with context %}
+{% set ISAIRGAP = salt['pillar.get']('global:airgap', False) %}
+{% set managerupdates = salt['pillar.get']('global:managerupdate', 0) %}
+{% set role = grains.id.split('_') | last %}
+
+# from airgap state
+{% if ISAIRGAP and grains.os == 'CentOS' %}
+{% set MANAGER = salt['grains.get']('master') %}
+airgapyum:
+  file.managed:
+    - name: /etc/yum/yum.conf
+    - source: salt://repo/client/files/centos/airgap/yum.conf
+
+airgap_repo:
+  pkgrepo.managed:
+    - humanname: Airgap Repo
+    - baseurl: https://{{ MANAGER }}/repo
+    - gpgcheck: 0
+    - sslverify: 0
+
+{% endif %}
+
+# from airgap and common
+{% if ABSENTFILES|length > 0%}
+  {% for file in ABSENTFILES  %}
+{{ file }}:
+  file.absent:
+    - name: {{ REPOPATH }}{{ file }}
+    - onchanges_in:
+      - cmd: cleanyum
+  {% endfor %}
+{% endif %}
+
+# from common state
+# Remove default Repos
+{% if grains['os'] == 'CentOS' %}
+repair_yumdb:
+  cmd.run:
+    - name: 'mv -f /var/lib/rpm/__db* /tmp && yum clean all'
+    - onlyif:
+      - 'yum check-update 2>&1 | grep "Error: rpmdb open failed"'
+
+crsynckeys:
+  file.recurse:
+    - name: /etc/pki/rpm_gpg
+    - source: salt://repo/client/files/centos/keys/
+
+{% if not ISAIRGAP %}
+    {% if role in ['eval', 'standalone', 'import', 'manager', 'managersearch'] or managerupdates == 0 %}
+remove_securityonionrepocache:
+  file.absent:
+    - name: /etc/yum.repos.d/securityonioncache.repo
+    {% endif %}
+
+    {% if role not in ['eval', 'standalone', 'import', 'manager', 'managersearch'] and managerupdates == 1 %}
+remove_securityonionrepo:
+  file.absent:
+    - name: /etc/yum.repos.d/securityonion.repo
+    {% endif %}
+
+crsecurityonionrepo:
+  file.managed:
+    {% if role in ['eval', 'standalone', 'import', 'manager', 'managersearch'] or managerupdates == 0 %}
+    - name: /etc/yum.repos.d/securityonion.repo
+    - source: salt://repo/client/files/centos/securityonion.repo
+    {% else %}
+    - name: /etc/yum.repos.d/securityonioncache.repo
+    - source: salt://repo/client/files/centos/securityonioncache.repo
+    {% endif %}
+    - mode: 644
+
+yumconf:
+  file.managed:
+    - name: /etc/yum.conf
+    - source: salt://repo/client/files/centos/yum.conf.jinja
+    - mode: 644
+    - template: jinja
+    - show_changes: False
+
+cleanairgap:
+  file.absent:
+    - name: /etc/yum.repos.d/airgap_repo.repo
+{% endif %}
+
+cleanyum:
+  cmd.run:
+    - name: 'yum clean metadata'
+    - onchanges:
+{% if ISAIRGAP %}
+      - file: airgapyum
+      - pkgrepo: airgap_repo
+{% else %}
+      - file: crsecurityonionrepo
+      - file: yumconf
+{% endif %}
+
+{% endif %}

salt/repo/client/init.sls

@@ -1,98 +1,2 @@
-{% from 'repo/client/map.jinja' import ABSENTFILES with context %}
-{% from 'repo/client/map.jinja' import REPOPATH with context %}
-{% set ISAIRGAP = salt['pillar.get']('global:airgap', False) %}
-{% set managerupdates = salt['pillar.get']('global:managerupdate', 0) %}
-{% set role = grains.id.split('_') | last %}
-
-# from airgap state
-{% if ISAIRGAP and grains.os == 'CentOS' %}
-{% set MANAGER = salt['grains.get']('master') %}
-airgapyum:
-  file.managed:
-    - name: /etc/yum/yum.conf
-    - source: salt://repo/client/files/centos/airgap/yum.conf
-
-airgap_repo:
-  pkgrepo.managed:
-    - humanname: Airgap Repo
-    - baseurl: https://{{ MANAGER }}/repo
-    - gpgcheck: 0
-    - sslverify: 0
-
-{% endif %}
-
-# from airgap and common
-{% if ABSENTFILES|length > 0%}
-  {% for file in ABSENTFILES  %}
-{{ file }}:
-  file.absent:
-    - name: {{ REPOPATH }}{{ file }}
-    - onchanges_in:
-      - cmd: cleanyum
-  {% endfor %}
-{% endif %}
-
-# from common state
-# Remove default Repos
-{% if grains['os'] == 'CentOS' %}
-repair_yumdb:
-  cmd.run:
-    - name: 'mv -f /var/lib/rpm/__db* /tmp && yum clean all'
-    - onlyif:
-      - 'yum check-update 2>&1 | grep "Error: rpmdb open failed"'
-
-crsynckeys:
-  file.recurse:
-    - name: /etc/pki/rpm-gpg
-    - source: salt://repo/client/files/centos/keys/
-
-{% if not ISAIRGAP %}
-    {% if role in ['eval', 'standalone', 'import', 'manager', 'managersearch'] or managerupdates == 0 %}
-remove_securityonionrepocache:
-  file.absent:
-    - name: /etc/yum.repos.d/securityonioncache.repo
-    {% endif %}
-
-    {% if role not in ['eval', 'standalone', 'import', 'manager', 'managersearch'] and managerupdates == 1 %}
-remove_securityonionrepo:
-  file.absent:
-    - name: /etc/yum.repos.d/securityonion.repo
-    {% endif %}
-
-crsecurityonionrepo:
-  file.managed:
-    {% if role in ['eval', 'standalone', 'import', 'manager', 'managersearch'] or managerupdates == 0 %}
-    - name: /etc/yum.repos.d/securityonion.repo
-    - source: salt://repo/client/files/centos/securityonion.repo
-    {% else %}
-    - name: /etc/yum.repos.d/securityonioncache.repo
-    - source: salt://repo/client/files/centos/securityonioncache.repo
-    {% endif %}
-    - mode: 644
-
-yumconf:
-  file.managed:
-    - name: /etc/yum.conf
-    - source: salt://repo/client/files/centos/yum.conf.jinja
-    - mode: 644
-    - template: jinja
-    - show_changes: False
-
-cleanairgap:
-  file.absent:
-    - name: /etc/yum.repos.d/airgap_repo.repo
-{% endif %}
-
-cleanyum:
-  cmd.run:
-    - name: 'yum clean metadata'
-    - onchanges:
-{% if ISAIRGAP %}
-      - file: airgapyum
-      - pkgrepo: airgap_repo
-{% else %}
-      - file: crsecurityonionrepo
-      - file: yumconf
-{% endif %}
-
-{% endif %}
+include:
+  - repo.client.{{grains.os | lower}}

salt/repo/client/ubuntu.sls

@@ -0,0 +1,20 @@
+# this removes the repo file left by bootstrap-salt.sh without -r
+remove_salt.list:
+  file.absent:
+    - name: /etc/apt/sources.list.d/salt.list
+
+saltstack.list:
+  file.managed:
+    - name: /etc/apt/sources.list.d/saltstack.list
+    - contents:
+      - deb https://repo.securityonion.net/file/securityonion-repo/ubuntu/{{grains.osrelease}}/amd64/salt/ {{grains.oscodename}} main
+
+apt_update:
+  cmd.run:
+    - name: apt-get update
+    - onchanges:
+      - file: saltstack.list
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 30

salt/salt/map.jinja

@@ -31,7 +31,7 @@
   {% if grains.os|lower in ['centos', 'redhat'] %}
       {% set UPGRADECOMMAND = 'yum clean all ; /usr/sbin/bootstrap-salt.sh -s 120 -r -F -x python3 stable ' ~ SALTVERSION %}
   {% elif grains.os|lower == 'ubuntu' %}
-    {% set UPGRADECOMMAND = '/usr/sbin/bootstrap-salt.sh -s 120 -F -x python3 stable ' ~ SALTVERSION %}
+    {% set UPGRADECOMMAND = '/usr/sbin/bootstrap-salt.sh -s 120 -r -F -x python3 stable ' ~ SALTVERSION %}
   {% endif %}
 {% else %}
   {% set UPGRADECOMMAND = 'echo Already running Salt Minion version ' ~ SALTVERSION %}

salt/salt/master.defaults.yaml

@@ -2,4 +2,4 @@
 # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and saltify function in so-functions
 salt:
   master:
-    version: 3004
+    version: 3004.1

salt/salt/minion.defaults.yaml

@@ -2,6 +2,6 @@
 # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and saltify function in so-functions
 salt:
   minion:
-    version: 3004
+    version: 3004.1
     check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default
     service_start_delay: 30 # in seconds.

salt/salt/minion.sls

@@ -32,6 +32,22 @@ install_salt_minion:
         exec 1>&- # close stdout
         exec 2>&- # close stderr
         nohup /bin/sh -c '{{ UPGRADECOMMAND }}' &
+
+  {# if we are the salt master #}
+  {% if grains.id.split('_')|first == grains.master %}
+remove_influxdb_continuous_query_state_file:
+  file.absent:
+    - name: /opt/so/state/influxdb_continuous_query.py.patched
+
+remove_influxdbmod_state_file:
+  file.absent:
+    - name: /opt/so/state/influxdbmod.py.patched
+
+remove_influxdb_retention_policy_state_file:
+  file.absent:
+    - name: /opt/so/state/influxdb_retention_policy.py.patched
+  {% endif %}
+
 {% endif %}
 
 {% if INSTALLEDSALTVERSION|string == SALTVERSION|string %}

salt/top.sls

@@ -20,16 +20,15 @@ base:
 
   '*':
     - cron.running
+    - repo.client
 
   'not G@saltversion:{{saltversion}}':
     - match: compound
     - salt.minion-state-apply-test
-    - repo.client
     - salt.minion
 
   'G@os:CentOS and G@saltversion:{{saltversion}}':
     - match: compound
-    - repo.client
     - yum.packages
 
   '* and G@saltversion:{{saltversion}}':