From 0ff5541801ccd556e419ce64920e72a0d2851913 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 7 Nov 2019 17:27:38 -0500 Subject: [PATCH 1/7] upgrade salt to 2019.2.2 - https://github.com/Security-Onion-Solutions/securityonion-saltstack/issues/105 --- so-setup-network.sh | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index de7b6f137..d3ae360f1 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -500,7 +500,7 @@ install_master() { wget --inet4-only -O /opt/so/gpg/GPG-KEY-WAZUH https://packages.wazuh.com/key/GPG-KEY-WAZUH else - apt-get install -y salt-common=2018.3.4+ds-1 salt-master=2018.3.4+ds-1 salt-minion=2018.3.4+ds-1 python-m2crypto + apt-get install -y salt-common=2019.2.2+ds-1 salt-master=2019.2.2+ds-1 salt-minion=2019.2.2+ds-1 python-m2crypto apt-mark hold salt-common salt-master salt-minion apt-get install -y python-m2crypto fi @@ -697,8 +697,8 @@ saltify() { if [ $INSTALLTYPE == 'MASTERONLY' ] || [ $INSTALLTYPE == 'EVALMODE' ]; then yum -y install https://repo.saltstack.com/yum/redhat/salt-repo-latest-2.el7.noarch.rpm - cp /etc/yum.repos.d/salt-latest.repo /etc/yum.repos.d/salt-2018-3.repo - sed -i 's/latest/2018.3/g' /etc/yum.repos.d/salt-2018-3.repo + cp /etc/yum.repos.d/salt-latest.repo /etc/yum.repos.d/salt-2019-2.repo + sed -i 's/latest/2019.2/g' /etc/yum.repos.d/salt-2019-2.repo cat > /etc/yum.repos.d/wazuh.repo <<\EOF [wazuh_repo] gpgcheck=1 @@ -812,13 +812,13 @@ EOF echo "gpgkey=file:///etc/pki/rpm-gpg/saltstack-signing-key" >> /etc/yum.repos.d/salt-latest.repo # Proxy is hating on me.. Lets just set it manually - echo "[salt-2018.3]" > /etc/yum.repos.d/salt-2018-3.repo - echo "name=SaltStack Latest Release Channel for RHEL/Centos \$releasever" >> /etc/yum.repos.d/salt-2018-3.repo - echo "baseurl=https://repo.saltstack.com/yum/redhat/7/\$basearch/2018.3" >> /etc/yum.repos.d/salt-2018-3.repo - echo "failovermethod=priority" >> /etc/yum.repos.d/salt-2018-3.repo - echo "enabled=1" >> /etc/yum.repos.d/salt-2018-3.repo - echo "gpgcheck=1" >> /etc/yum.repos.d/salt-2018-3.repo - echo "gpgkey=file:///etc/pki/rpm-gpg/saltstack-signing-key" >> /etc/yum.repos.d/salt-2018-3.repo + echo "[salt-2019.2]" > /etc/yum.repos.d/salt-2019-2.repo + echo "name=SaltStack Latest Release Channel for RHEL/Centos \$releasever" >> /etc/yum.repos.d/salt-2019-2.repo + echo "baseurl=https://repo.saltstack.com/yum/redhat/7/\$basearch/2019.2" >> /etc/yum.repos.d/salt-2019-2.repo + echo "failovermethod=priority" >> /etc/yum.repos.d/salt-2019-2.repo + echo "enabled=1" >> /etc/yum.repos.d/salt-2019-2.repo + echo "gpgcheck=1" >> /etc/yum.repos.d/salt-2019-2.repo + echo "gpgkey=file:///etc/pki/rpm-gpg/saltstack-signing-key" >> /etc/yum.repos.d/salt-2019-2.repo cat > /etc/yum.repos.d/wazuh.repo <<\EOF [wazuh_repo] @@ -831,8 +831,8 @@ protect=1 EOF else yum -y install https://repo.saltstack.com/yum/redhat/salt-repo-latest-2.el7.noarch.rpm - cp /etc/yum.repos.d/salt-latest.repo /etc/yum.repos.d/salt-2018-3.repo - sed -i 's/latest/2018.3/g' /etc/yum.repos.d/salt-2018-3.repo + cp /etc/yum.repos.d/salt-latest.repo /etc/yum.repos.d/salt-2019-2.repo + sed -i 's/latest/2019.2/g' /etc/yum.repos.d/salt-2019-2.repo cat > /etc/yum.repos.d/wazuh.repo <<\EOF [wazuh_repo] gpgcheck=1 @@ -846,16 +846,16 @@ EOF fi yum clean expire-cache - yum -y install salt-minion-2018.3.4 yum-utils device-mapper-persistent-data lvm2 openssl python-dateutil + yum -y install salt-minion-2019.2.2 yum-utils device-mapper-persistent-data lvm2 openssl python-dateutil yum -y update exclude=salt* systemctl enable salt-minion # Nasty hack but required for now if [ $INSTALLTYPE == 'MASTERONLY' ] || [ $INSTALLTYPE == 'EVALMODE' ]; then - yum -y install salt-master-2018.3.4 python-m2crypto salt-minion-2018.3.4 m2crypto + yum -y install salt-master-2019.2.2 python-m2crypto salt-minion-2019.2.2 m2crypto systemctl enable salt-master else - yum -y install salt-minion-2018.3.4 python-m2m2crypto m2crypto + yum -y install salt-minion-2019.2.2 python-m2m2crypto m2crypto fi echo "exclude=salt*" >> /etc/yum.conf @@ -874,9 +874,9 @@ EOF # Install the repo for salt wget --inet4-only -O - https://repo.saltstack.com/apt/ubuntu/$UVER/amd64/latest/SALTSTACK-GPG-KEY.pub | apt-key add - - wget --inet4-only -O - https://repo.saltstack.com/apt/ubuntu/$UVER/amd64/2018.3/SALTSTACK-GPG-KEY.pub | apt-key add - + wget --inet4-only -O - https://repo.saltstack.com/apt/ubuntu/$UVER/amd64/2019.2/SALTSTACK-GPG-KEY.pub | apt-key add - echo "deb http://repo.saltstack.com/apt/ubuntu/$UVER/amd64/latest xenial main" > /etc/apt/sources.list.d/saltstack.list - echo "deb http://repo.saltstack.com/apt/ubuntu/$UVER/amd64/2018.3 xenial main" > /etc/apt/sources.list.d/saltstack2018.list + echo "deb http://repo.saltstack.com/apt/ubuntu/$UVER/amd64/2019.2 xenial main" > /etc/apt/sources.list.d/saltstack2019.list # Lets get the docker repo added curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - @@ -895,7 +895,7 @@ EOF # Initialize the new repos apt-get update >> $SETUPLOG 2>&1 - apt-get -y install salt-minion=2018.3.4+ds-1 salt-common=2018.3.4+ds-1 python-m2cryptoi python-dateutil >> $SETUPLOG 2>&1 + apt-get -y install salt-minion=2019.2.2+ds-1 salt-common=2019.2.2+ds-1 python-m2cryptoi python-dateutil >> $SETUPLOG 2>&1 apt-mark hold salt-minion salt-common else @@ -909,7 +909,7 @@ EOF echo "deb https://packages.wazuh.com/3.x/apt/ stable main" | tee /etc/apt/sources.list.d/wazuh.list # Initialize the new repos apt-get update >> $SETUPLOG 2>&1 - apt-get -y install salt-minion=2018.3.4+ds-1 salt-common=2018.3.4+ds-1 python-m2crypto python-dateutil >> $SETUPLOG 2>&1 + apt-get -y install salt-minion=2019.2.2+ds-1 salt-common=2019.2.2+ds-1 python-m2crypto python-dateutil >> $SETUPLOG 2>&1 apt-mark hold salt-minion salt-common fi From 939ab918b4778ebd3621c7a351e3760ffa0d69df Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 7 Nov 2019 17:31:06 -0500 Subject: [PATCH 2/7] update states using module.run - https://github.com/Security-Onion-Solutions/securityonion-saltstack/issues/106 --- salt/ca/init.sls | 8 ++++---- salt/patch/os/schedule.sls | 8 ++++---- salt/playbook/init.sls | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/salt/ca/init.sls b/salt/ca/init.sls index 27344fc7f..407516f6e 100644 --- a/salt/ca/init.sls +++ b/salt/ca/init.sls @@ -39,10 +39,10 @@ pki_private_key: - require: - file: /etc/pki -mine.send: +send_x509_pem_entries_to_mine: module.run: - - func: x509.get_pem_entries - - kwargs: - glob_path: /etc/pki/ca.crt + - mine.send: + - func: x509.get_pem_entries + - glob_path: /etc/pki/ca.crt - onchanges: - x509: /etc/pki/ca.crt diff --git a/salt/patch/os/schedule.sls b/salt/patch/os/schedule.sls index 128862fa7..62232b0d1 100644 --- a/salt/patch/os/schedule.sls +++ b/salt/patch/os/schedule.sls @@ -64,13 +64,13 @@ remove_patch_os_schedule: {% else %} -no_os_patch_schedule_name_set: +no_patch_os_schedule_name_set: test.fail_without_changes: - name: "Set a pillar value for patch:os:schedule_name in this minion's .sls file. If an OS patch schedule is not listed as enabled in show_schedule output below, then OS patches will need to be applied manually until this is corrected." -show_schedule: +show_patch_os_schedule: module.run: - - name: schedule.is_enabled - - m_name: patch_os_schedule + - schedule.is_enabled: + - name: patch_os_schedule {% endif %} diff --git a/salt/playbook/init.sls b/salt/playbook/init.sls index ef66966f3..6a054195a 100644 --- a/salt/playbook/init.sls +++ b/salt/playbook/init.sls @@ -11,9 +11,9 @@ playbookdb: playbookwebhook: module.run: - - name: sqlite3.modify - - db: /opt/so/conf/playbook/redmine.db - - sql: "update webhooks set url = 'http://{{MASTERIP}}:7000/playbook/webhook' where project_id = 1" + - sqlite3.modify: + - db: /opt/so/conf/playbook/redmine.db + - sql: "update webhooks set url = 'http://{{MASTERIP}}:7000/playbook/webhook' where project_id = 1" navigatorconfig: file.managed: From f58031f4e3ed601a85282de6014013375cebc276 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 7 Nov 2019 17:35:19 -0500 Subject: [PATCH 3/7] enable the new module.run behavior on the minion config - https://github.com/Security-Onion-Solutions/securityonion-saltstack/issues/106 --- so-setup-network.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/so-setup-network.sh b/so-setup-network.sh index d3ae360f1..f7f6c7bd9 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -255,6 +255,9 @@ configure_minion() { fi + echo "use_superseded:" >> /etc/salt/minion + echo " - module.run" >> /etc/salt/minion + service salt-minion restart } From aac9ab8e83f4f54b89a9e3050eceb7e6e576d410 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 11 Nov 2019 08:49:59 -0500 Subject: [PATCH 4/7] install the py3 version of salt 2019.2.2 - https://github.com/Security-Onion-Solutions/securityonion-saltstack/issues/105 --- so-setup-network.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index f7f6c7bd9..b7c858d32 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -699,7 +699,7 @@ saltify() { ADDUSER=adduser if [ $INSTALLTYPE == 'MASTERONLY' ] || [ $INSTALLTYPE == 'EVALMODE' ]; then - yum -y install https://repo.saltstack.com/yum/redhat/salt-repo-latest-2.el7.noarch.rpm + yum -y install https://repo.saltstack.com/py3/redhat/salt-py3-repo-latest-2.el7.noarch.rpm cp /etc/yum.repos.d/salt-latest.repo /etc/yum.repos.d/salt-2019-2.repo sed -i 's/latest/2019.2/g' /etc/yum.repos.d/salt-2019-2.repo cat > /etc/yum.repos.d/wazuh.repo <<\EOF @@ -878,8 +878,8 @@ EOF # Install the repo for salt wget --inet4-only -O - https://repo.saltstack.com/apt/ubuntu/$UVER/amd64/latest/SALTSTACK-GPG-KEY.pub | apt-key add - wget --inet4-only -O - https://repo.saltstack.com/apt/ubuntu/$UVER/amd64/2019.2/SALTSTACK-GPG-KEY.pub | apt-key add - - echo "deb http://repo.saltstack.com/apt/ubuntu/$UVER/amd64/latest xenial main" > /etc/apt/sources.list.d/saltstack.list - echo "deb http://repo.saltstack.com/apt/ubuntu/$UVER/amd64/2019.2 xenial main" > /etc/apt/sources.list.d/saltstack2019.list + echo "deb http://repo.saltstack.com/py3/ubuntu/$UVER/amd64/latest xenial main" > /etc/apt/sources.list.d/saltstack.list + echo "deb http://repo.saltstack.com/py3/ubuntu/$UVER/amd64/2019.2 xenial main" > /etc/apt/sources.list.d/saltstack2019.list # Lets get the docker repo added curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - From bd26db1f239eea0ad7f6980e32d1a01fe86aaea7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 11 Nov 2019 10:50:35 -0500 Subject: [PATCH 5/7] install docker python3 library and set env var PYTHONPATH - https://github.com/Security-Onion-Solutions/securityonion-saltstack/issues/105 --- so-setup-network.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index b7c858d32..8c9efa955 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -355,7 +355,9 @@ docker_install() { yum -y install yum-utils device-mapper-persistent-data lvm2 openssl yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo yum -y update - yum -y install docker-ce docker-python python-docker + yum -y install docker-ce + pip3 install docker + set_environment_var "PYTHONPATH=/usr/local/lib/python3.6/site-packages/" if [ $INSTALLTYPE != 'EVALMODE' ]; then docker_registry fi @@ -1023,6 +1025,15 @@ sensor_pillar() { } +set_environment_var() { + + echo "Setting environment variable: $1" + + export "$1" + echo "$1" >> /etc/environment + +} + set_hostname() { hostnamectl set-hostname --static $HOSTNAME From 67108ccc77da9e710c42eccbd812473060274610 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 11 Nov 2019 11:09:32 -0500 Subject: [PATCH 6/7] iteritems deprecated for items in py3 - https://github.com/Security-Onion-Solutions/securityonion-saltstack/issues/105 --- salt/common/init.sls | 8 ++++---- salt/patch/os/schedule.sls | 2 +- salt/utility/bin/crossthestreams.sh | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 1bba4c871..93940669c 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -316,7 +316,7 @@ grafanaconf: - source: salt://common/grafana/etc {% if salt['pillar.get']('mastertab', False) %} -{%- for SN, SNDATA in salt['pillar.get']('mastertab', {}).iteritems() %} +{%- for SN, SNDATA in salt['pillar.get']('mastertab', {}).items() %} dashboard-master: file.managed: - name: /opt/so/conf/grafana/grafana_dashboards/master/{{ SN }}-Master.json @@ -337,7 +337,7 @@ dashboard-master: {% endif %} {% if salt['pillar.get']('sensorstab', False) %} -{%- for SN, SNDATA in salt['pillar.get']('sensorstab', {}).iteritems() %} +{%- for SN, SNDATA in salt['pillar.get']('sensorstab', {}).items() %} dashboard-{{ SN }}: file.managed: - name: /opt/so/conf/grafana/grafana_dashboards/forward_nodes/{{ SN }}-Sensor.json @@ -358,7 +358,7 @@ dashboard-{{ SN }}: {% endif %} {% if salt['pillar.get']('nodestab', False) %} -{%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).iteritems() %} +{%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} dashboard-{{ SN }}: file.managed: - name: /opt/so/conf/grafana/grafana_dashboards/storage_nodes/{{ SN }}-Node.json @@ -379,7 +379,7 @@ dashboard-{{ SN }}: {% endif %} {% if salt['pillar.get']('evaltab', False) %} -{%- for SN, SNDATA in salt['pillar.get']('evaltab', {}).iteritems() %} +{%- for SN, SNDATA in salt['pillar.get']('evaltab', {}).items() %} dashboard-{{ SN }}: file.managed: - name: /opt/so/conf/grafana/grafana_dashboards/eval/{{ SN }}-Node.json diff --git a/salt/patch/os/schedule.sls b/salt/patch/os/schedule.sls index 62232b0d1..a91e61dfe 100644 --- a/salt/patch/os/schedule.sls +++ b/salt/patch/os/schedule.sls @@ -15,7 +15,7 @@ patch_os_schedule: - patch.os - when: {% for days in os_schedule.patch.os.schedule %} - {% for day, times in days.iteritems() %} + {% for day, times in days.items() %} {% for time in times %} - {{day}} {{time}} {% endfor %} diff --git a/salt/utility/bin/crossthestreams.sh b/salt/utility/bin/crossthestreams.sh index b9c8f6c1d..3cd8b005c 100644 --- a/salt/utility/bin/crossthestreams.sh +++ b/salt/utility/bin/crossthestreams.sh @@ -31,6 +31,6 @@ echo "Applying cross cluster search config..." # Add all the storage nodes to cross cluster searching. -{%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).iteritems() %} +{%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} curl -XPUT http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SNDATA.ip }}:9300"]}}}}}' {%- endfor %} From f1d7dff5657f05a185afa5b075974d81944ff361 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 11 Nov 2019 12:08:37 -0500 Subject: [PATCH 7/7] set PYTHONPATH env var differently - https://github.com/Security-Onion-Solutions/securityonion-saltstack/issues/105 --- so-setup-network.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index 8c9efa955..54948c538 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -357,7 +357,7 @@ docker_install() { yum -y update yum -y install docker-ce pip3 install docker - set_environment_var "PYTHONPATH=/usr/local/lib/python3.6/site-packages/" + set_environment_var "PYTHONPATH=$PYTHONPATH:/usr/local/lib/python3.6/site-packages/" if [ $INSTALLTYPE != 'EVALMODE' ]; then docker_registry fi @@ -1030,7 +1030,7 @@ set_environment_var() { echo "Setting environment variable: $1" export "$1" - echo "$1" >> /etc/environment + echo "export $1" >> /etc/profile.d/set_env_vars.sh }