diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 43fdb8e01..cbc0bd4e5 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -50,4 +50,4 @@ check_password() { local password=$1 echo "$password" | egrep -v "'|\"|\\$|\\\\" > /dev/null 2>&1 return $? -} \ No newline at end of file +} diff --git a/salt/common/tools/sbin/so-docker-refresh b/salt/common/tools/sbin/so-docker-refresh index 770d9f241..b39513990 100755 --- a/salt/common/tools/sbin/so-docker-refresh +++ b/salt/common/tools/sbin/so-docker-refresh @@ -16,6 +16,7 @@ # along with this program. If not, see . . /usr/sbin/so-common +. /usr/sbin/so-image-common manager_check() { # Check to see if this is a manager @@ -28,21 +29,6 @@ manager_check() { fi } -update_docker_containers() { - - # Download the containers from the interwebs - for i in "${TRUSTED_CONTAINERS[@]}" - do - # Pull down the trusted docker image - echo "Downloading $i" - docker pull --disable-content-trust=false docker.io/$IMAGEREPO/$i - # Tag it with the new registry destination - docker tag $IMAGEREPO/$i $HOSTNAME:5000/$IMAGEREPO/$i - docker push $HOSTNAME:5000/$IMAGEREPO/$i - done - -} - version_check() { if [ -f /etc/soversion ]; then VERSION=$(cat /etc/soversion) @@ -58,54 +44,5 @@ version_check # Use the hostname HOSTNAME=$(hostname) # List all the containers -if [ $MANAGERCHECK != 'so-helix' ]; then - TRUSTED_CONTAINERS=( \ - "so-acng:$VERSION" \ - "so-thehive-cortex:$VERSION" \ - "so-curator:$VERSION" \ - "so-domainstats:$VERSION" \ - "so-elastalert:$VERSION" \ - "so-elasticsearch:$VERSION" \ - "so-filebeat:$VERSION" \ - "so-fleet:$VERSION" \ - "so-fleet-launcher:$VERSION" \ - "so-freqserver:$VERSION" \ - "so-grafana:$VERSION" \ - "so-idstools:$VERSION" \ - "so-influxdb:$VERSION" \ - "so-kibana:$VERSION" \ - "so-kratos:$VERSION" \ - "so-logstash:$VERSION" \ - "so-minio:$VERSION" \ - "so-mysql:$VERSION" \ - "so-nginx:$VERSION" \ - "so-pcaptools:$VERSION" \ - "so-playbook:$VERSION" \ - "so-redis:$VERSION" \ - "so-soc:$VERSION" \ - "so-soctopus:$VERSION" \ - "so-steno:$VERSION" \ - "so-strelka-frontend:$VERSION" \ - "so-strelka-manager:$VERSION" \ - "so-strelka-backend:$VERSION" \ - "so-strelka-filestream:$VERSION" \ - "so-suricata:$VERSION" \ - "so-telegraf:$VERSION" \ - "so-thehive:$VERSION" \ - "so-thehive-es:$VERSION" \ - "so-wazuh:$VERSION" \ - "so-zeek:$VERSION" ) - else - TRUSTED_CONTAINERS=( \ - "so-filebeat:$VERSION" \ - "so-idstools:$VERSION" \ - "so-logstash:$VERSION" \ - "so-nginx:$VERSION" \ - "so-redis:$VERSION" \ - "so-steno:$VERSION" \ - "so-suricata:$VERSION" \ - "so-telegraf:$VERSION" \ - "so-zeek:$VERSION" ) - fi - +container_list update_docker_containers diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common new file mode 100755 index 000000000..46f2d4a0f --- /dev/null +++ b/salt/common/tools/sbin/so-image-common @@ -0,0 +1,133 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Figure out if this is soup or refresh +if [ -z "$VERSION" ]; then + VERSION="$NEWVERSION" +fi + +container_list() { + MANAGERCHECK=$(cat /etc/salt/grains | grep role | awk '{print $2}') + if [ $MANAGERCHECK == 'so-import' ]; then + TRUSTED_CONTAINERS=( \ + "so-idstools" \ + "so-nginx" \ + "so-filebeat" \ + "so-suricata" \ + "so-soc" \ + "so-elasticsearch" \ + "so-kibana" \ + "so-kratos" \ + "so-suricata" \ + "so-registry" \ + "so-pcaptools" \ + "so-zeek" ) + elif [ $MANAGERCHECK != 'so-helix' ]; then + TRUSTED_CONTAINERS=( \ + "so-acng" \ + "so-thehive-cortex" \ + "so-curator" \ + "so-domainstats" \ + "so-elastalert" \ + "so-elasticsearch" \ + "so-filebeat" \ + "so-fleet" \ + "so-fleet-launcher" \ + "so-freqserver" \ + "so-grafana" \ + "so-idstools" \ + "so-influxdb" \ + "so-kibana" \ + "so-kratos" \ + "so-logstash" \ + "so-minio" \ + "so-mysql" \ + "so-nginx" \ + "so-pcaptools" \ + "so-playbook" \ + "so-redis" \ + "so-soc" \ + "so-soctopus" \ + "so-steno" \ + "so-strelka-frontend" \ + "so-strelka-manager" \ + "so-strelka-backend" \ + "so-strelka-filestream" \ + "so-suricata" \ + "so-telegraf" \ + "so-thehive" \ + "so-thehive-es" \ + "so-wazuh" \ + "so-zeek" ) + else + TRUSTED_CONTAINERS=( \ + "so-filebeat" \ + "so-idstools" \ + "so-logstash" \ + "so-nginx" \ + "so-redis" \ + "so-steno" \ + "so-suricata" \ + "so-telegraf" \ + "so-zeek" ) + fi +} + +update_docker_containers() { + # Let's make sure we have the public key + curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS | gpg --import - + + CONTAINER_REGISTRY=quay.io + SIGNPATH=/root/sosigs + rm -rf $SIGNPATH + mkdir -p $SIGNPATH + if [ -z "$BRANCH" ]; then + BRANCH="master" + fi + # Download the containers from the interwebs + for i in "${TRUSTED_CONTAINERS[@]}" + do + # Pull down the trusted docker image + echo "Downloading $i" + docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION + + # Get signature + curl https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/$BRANCH/sigs/images/$VERSION/$i.sig --output $SIGNPATH/$i.sig + if [[ $? -ne 0 ]]; then + echo "Unable to pull signature file for $i:$VERSION" + exit 1 + fi + # Dump our hash values + docker inspect $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION | jq '.[0].Created, .[0].RepoDigests, .[0].RootFS.Layers' > $SIGNPATH/$i.txt + if [[ $? -ne 0 ]]; then + echo "Unable to inspect $i:$VERSION" + exit 1 + fi + GPGTEST=$(gpg --verify $SIGNPATH/$i.sig $SIGNPATH/$i.txt 2>&1) + if [[ $? -eq 0 ]]; then + # Tag it with the new registry destination + docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION $HOSTNAME:5000/$IMAGEREPO/$i:$VERSION + docker push $HOSTNAME:5000/$IMAGEREPO/$i:$VERSION + else + echo "There is a problem downloading the $i:$VERSION image. Details: " + echo "" + echo $GPGTEST + exit 1 + fi + done + +} \ No newline at end of file diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index c75f89255..538ac1c56 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -16,6 +16,7 @@ # along with this program. If not, see . . /usr/sbin/so-common +. /usr/sbin/so-image-common UPDATE_DIR=/tmp/sogh/securityonion INSTALLEDVERSION=$(cat /etc/soversion) INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk {'print $2'}) @@ -79,6 +80,24 @@ airgap_mounted() { fi } +airgap_update_dockers() { + if [ $is_airgap -eq 0 ]; then + # Let's copy the tarball + if [ ! -f $AGDOCKER/registry.tar ]; then + echo "Unable to locate registry. Exiting" + exit 1 + else + echo "Stopping the registry docker" + docker stop so-dockerregistry + docker rm so-dockerregistry + echo "Copying the new dockers over" + tar xvf $AGDOCKER/registry.tar -C /nsm/docker-registry/docker + echo "Add Registry back" + docker load -i $AGDOCKER/registry_image.tar + fi + +} + check_airgap() { # See if this is an airgap install AIRGAP=$(cat /opt/so/saltstack/local/pillar/global.sls | grep airgap | awk '{print $2}') @@ -290,103 +309,6 @@ update_centos_repo() { createrepo /nsm/repo } -update_dockers() { - if [ $is_airgap -eq 0 ]; then - # Let's copy the tarball - if [ ! -f $AGDOCKER/registry.tar ]; then - echo "Unable to locate registry. Exiting" - exit 0 - else - echo "Stopping the registry docker" - docker stop so-dockerregistry - docker rm so-dockerregistry - echo "Copying the new dockers over" - tar xvf $AGDOCKER/registry.tar -C /nsm/docker-registry/docker - fi - else - # List all the containers - if [ $MANAGERCHECK == 'so-import' ]; then - TRUSTED_CONTAINERS=( \ - "so-idstools" \ - "so-nginx" \ - "so-filebeat" \ - "so-suricata" \ - "so-soc" \ - "so-elasticsearch" \ - "so-kibana" \ - "so-kratos" \ - "so-suricata" \ - "so-registry" \ - "so-pcaptools" \ - "so-zeek" ) - elif [ $MANAGERCHECK != 'so-helix' ]; then - TRUSTED_CONTAINERS=( \ - "so-acng" \ - "so-thehive-cortex" \ - "so-curator" \ - "so-domainstats" \ - "so-elastalert" \ - "so-elasticsearch" \ - "so-filebeat" \ - "so-fleet" \ - "so-fleet-launcher" \ - "so-freqserver" \ - "so-grafana" \ - "so-idstools" \ - "so-influxdb" \ - "so-kibana" \ - "so-kratos" \ - "so-logstash" \ - "so-minio" \ - "so-mysql" \ - "so-nginx" \ - "so-pcaptools" \ - "so-playbook" \ - "so-redis" \ - "so-soc" \ - "so-soctopus" \ - "so-steno" \ - "so-strelka-frontend" \ - "so-strelka-manager" \ - "so-strelka-backend" \ - "so-strelka-filestream" \ - "so-suricata" \ - "so-telegraf" \ - "so-thehive" \ - "so-thehive-es" \ - "so-wazuh" \ - "so-zeek" ) - else - TRUSTED_CONTAINERS=( \ - "so-filebeat" \ - "so-idstools" \ - "so-logstash" \ - "so-nginx" \ - "so-redis" \ - "so-steno" \ - "so-suricata" \ - "so-telegraf" \ - "so-zeek" ) - fi - -# Download the containers from the interwebs - for i in "${TRUSTED_CONTAINERS[@]}" - do - # Pull down the trusted docker image - echo "Downloading $i:$NEWVERSION" - docker pull --disable-content-trust=false docker.io/$IMAGEREPO/$i:$NEWVERSION - # Tag it with the new registry destination - docker tag $IMAGEREPO/$i:$NEWVERSION $HOSTNAME:5000/$IMAGEREPO/$i:$NEWVERSION - docker push $HOSTNAME:5000/$IMAGEREPO/$i:$NEWVERSION - done - fi - echo "Add Registry back if airgap" - if [ $is_airgap -eq 0 ]; then - docker load -i $AGDOCKER/registry_image.tar - fi - -} - update_version() { # Update the version to the latest echo "Updating the Security Onion version file." @@ -513,7 +435,12 @@ echo "" echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION." echo "" echo "Updating dockers to $NEWVERSION." -update_dockers +if [ $is_airgap -eq 0 ]; then + airgap_update_dockers +else + container_list + update_docker_containers +fi echo "" echo "Stopping Salt Minion service." systemctl stop salt-minion diff --git a/setup/so-functions b/setup/so-functions index 51a9b01c0..b8616439d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -19,6 +19,8 @@ source ./so-whiptail source ./so-variables source ./so-common-functions +CONTAINER_REGISTRY=quay.io + SOVERSION=$(cat ../VERSION) log() { @@ -874,79 +876,105 @@ docker_seed_registry() { if ! [ -f /nsm/docker-registry/docker/registry.tar ]; then if [ "$install_type" == 'IMPORT' ]; then local TRUSTED_CONTAINERS=(\ - "so-idstools:$VERSION" \ - "so-nginx:$VERSION" \ - "so-filebeat:$VERSION" \ - "so-suricata:$VERSION" \ - "so-soc:$VERSION" \ - "so-steno:$VERSION" \ - "so-elasticsearch:$VERSION" \ - "so-kibana:$VERSION" \ - "so-kratos:$VERSION" \ - "so-suricata:$VERSION" \ - "so-pcaptools:$VERSION" \ - "so-zeek:$VERSION" + "so-idstools" \ + "so-nginx" \ + "so-filebeat" \ + "so-suricata" \ + "so-soc" \ + "so-steno" \ + "so-elasticsearch" \ + "so-kibana" \ + "so-kratos" \ + "so-suricata" \ + "so-pcaptools" \ + "so-zeek" ) else local TRUSTED_CONTAINERS=(\ - "so-nginx:$VERSION" \ - "so-filebeat:$VERSION" \ - "so-logstash:$VERSION" \ - "so-idstools:$VERSION" \ - "so-redis:$VERSION" \ - "so-steno:$VERSION" \ - "so-suricata:$VERSION" \ - "so-telegraf:$VERSION" \ - "so-zeek:$VERSION" + "so-nginx" \ + "so-filebeat" \ + "so-logstash" \ + "so-idstools" \ + "so-redis" \ + "so-steno" \ + "so-suricata" \ + "so-telegraf" \ + "so-zeek" ) fi if [ "$install_type" != 'HELIXSENSOR' ] && [ "$install_type" != 'IMPORT' ]; then TRUSTED_CONTAINERS=("${TRUSTED_CONTAINERS[@]}" \ - "so-acng:$VERSION" \ - "so-thehive-cortex:$VERSION" \ - "so-curator:$VERSION" \ - "so-domainstats:$VERSION" \ - "so-elastalert:$VERSION" \ - "so-elasticsearch:$VERSION" \ - "so-fleet:$VERSION" \ - "so-fleet-launcher:$VERSION" \ - "so-freqserver:$VERSION" \ - "so-grafana:$VERSION" \ - "so-influxdb:$VERSION" \ - "so-kibana:$VERSION" \ - "so-minio:$VERSION" \ - "so-mysql:$VERSION" \ - "so-pcaptools:$VERSION" \ - "so-playbook:$VERSION" \ - "so-soc:$VERSION" \ - "so-kratos:$VERSION" \ - "so-soctopus:$VERSION" \ - "so-steno:$VERSION" \ - "so-strelka-frontend:$VERSION" \ - "so-strelka-manager:$VERSION" \ - "so-strelka-backend:$VERSION" \ - "so-strelka-filestream:$VERSION" \ - "so-thehive:$VERSION" \ - "so-thehive-es:$VERSION" \ - "so-wazuh:$VERSION" + "so-acng" \ + "so-thehive-cortex" \ + "so-curator" \ + "so-domainstats" \ + "so-elastalert" \ + "so-elasticsearch" \ + "so-fleet" \ + "so-fleet-launcher" \ + "so-freqserver" \ + "so-grafana" \ + "so-influxdb" \ + "so-kibana" \ + "so-minio" \ + "so-mysql" \ + "so-pcaptools" \ + "so-playbook" \ + "so-soc" \ + "so-kratos" \ + "so-soctopus" \ + "so-steno" \ + "so-strelka-frontend" \ + "so-strelka-manager" \ + "so-strelka-backend" \ + "so-strelka-filestream" \ + "so-thehive" \ + "so-thehive-es" \ + "so-wazuh" ) fi local percent=25 + # Let's make sure we have the public key + curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS | gpg --import - + + SIGNPATH=/root/sosigs + rm -rf $SIGNPATH + mkdir -p $SIGNPATH + if [ -z "$BRANCH" ]; then + BRANCH="master" + fi for i in "${TRUSTED_CONTAINERS[@]}"; do if [ "$install_type" != 'HELIXSENSOR' ]; then ((percent=percent+1)); else ((percent=percent+6)); fi # Pull down the trusted docker image - set_progress_str "$percent" "Downloading $i" + set_progress_str "$percent" "Downloading $i:$VERSION" { - - if ! docker pull --disable-content-trust=false docker.io/$IMAGEREPO/"$i"; then - sleep 5 - docker pull --disable-content-trust=false docker.io/$IMAGEREPO/"$i" - fi - # Tag it with the new registry destination - docker tag $IMAGEREPO/"$i" "$HOSTNAME":5000/$IMAGEREPO/"$i" - docker push "$HOSTNAME":5000/$IMAGEREPO/"$i" - #docker rmi $IMAGEREPO/"$i" - } >> "$setup_log" 2>&1 + echo "Downloading $i" + docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION + + # Get signature + curl https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/$BRANCH/sigs/images/$VERSION/$i.sig --output $SIGNPATH/$i.sig + if [[ $? -ne 0 ]]; then + echo "Unable to pull signature file for $i:$VERSION" + exit 1 + fi + # Dump our hash values + docker inspect $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION | jq '.[0].Created, .[0].RepoDigests, .[0].RootFS.Layers' > $SIGNPATH/$i.txt + if [[ $? -ne 0 ]]; then + echo "Unable to inspect $i" + exit 1 + fi + GPGTEST=$(gpg --verify $SIGNPATH/$i.sig $SIGNPATH/$i.txt 2>&1) + if [[ $? -eq 0 ]]; then + # Tag it with the new registry destination + docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION $HOSTNAME:5000/$IMAGEREPO/$i:$VERSION + docker push $HOSTNAME:5000/$IMAGEREPO/$i:$VERSION + else + echo "There is a problem downloading the $i image. Details: " + echo "" + echo $GPGTEST + exit 1 + fi + } >> "$setup_log" 2>&1 done else tar xvf /nsm/docker-registry/docker/registry.tar -C /nsm/docker-registry/docker >> "$setup_log" 2>&1