Merge pull request #1893 from Security-Onion-Solutions/gpg

GPG Docker Image Verification

salt/common/tools/sbin/so-docker-refresh

@@ -16,6 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 . /usr/sbin/so-common
+. /usr/sbin/so-image-common
 
 manager_check() {
   # Check to see if this is a manager
@@ -28,21 +29,6 @@ manager_check() {
   fi
 }
 
-update_docker_containers() {
-  
-  # Download the containers from the interwebs
-  for i in "${TRUSTED_CONTAINERS[@]}"
-  do
-    # Pull down the trusted docker image
-    echo "Downloading $i"
-    docker pull --disable-content-trust=false docker.io/$IMAGEREPO/$i
-    # Tag it with the new registry destination
-    docker tag $IMAGEREPO/$i $HOSTNAME:5000/$IMAGEREPO/$i
-    docker push $HOSTNAME:5000/$IMAGEREPO/$i
-  done
-  
-}
-
 version_check() {
   if [ -f /etc/soversion ]; then
     VERSION=$(cat /etc/soversion)
@@ -58,54 +44,5 @@ version_check
 # Use the hostname
 HOSTNAME=$(hostname)
 # List all the containers
-if [ $MANAGERCHECK != 'so-helix' ]; then
+container_list
-    TRUSTED_CONTAINERS=( \
-    "so-acng:$VERSION" \
-    "so-thehive-cortex:$VERSION" \
-    "so-curator:$VERSION" \
-    "so-domainstats:$VERSION" \
-    "so-elastalert:$VERSION" \
-    "so-elasticsearch:$VERSION" \
-    "so-filebeat:$VERSION" \
-    "so-fleet:$VERSION" \
-    "so-fleet-launcher:$VERSION" \
-    "so-freqserver:$VERSION" \
-    "so-grafana:$VERSION" \
-    "so-idstools:$VERSION" \
-    "so-influxdb:$VERSION" \
-    "so-kibana:$VERSION" \
-    "so-kratos:$VERSION" \
-    "so-logstash:$VERSION" \
-    "so-minio:$VERSION" \
-    "so-mysql:$VERSION" \
-    "so-nginx:$VERSION" \
-    "so-pcaptools:$VERSION" \
-    "so-playbook:$VERSION" \
-    "so-redis:$VERSION" \
-    "so-soc:$VERSION" \
-    "so-soctopus:$VERSION" \
-    "so-steno:$VERSION" \
-    "so-strelka-frontend:$VERSION" \
-		"so-strelka-manager:$VERSION" \
-		"so-strelka-backend:$VERSION" \
-		"so-strelka-filestream:$VERSION" \
-    "so-suricata:$VERSION" \
-    "so-telegraf:$VERSION" \
-    "so-thehive:$VERSION" \
-    "so-thehive-es:$VERSION" \
-    "so-wazuh:$VERSION" \
-    "so-zeek:$VERSION" )
-  else
-    TRUSTED_CONTAINERS=( \
-    "so-filebeat:$VERSION" \
-    "so-idstools:$VERSION" \
-    "so-logstash:$VERSION" \
-    "so-nginx:$VERSION" \
-    "so-redis:$VERSION" \
-    "so-steno:$VERSION" \
-    "so-suricata:$VERSION" \
-    "so-telegraf:$VERSION" \
-    "so-zeek:$VERSION" )
-  fi
-
 update_docker_containers

salt/common/tools/sbin/so-image-common

@@ -0,0 +1,133 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Figure out if this is soup or refresh
+if [ -z "$VERSION" ]; then
+    VERSION="$NEWVERSION"
+fi
+
+container_list() {
+    MANAGERCHECK=$(cat /etc/salt/grains | grep role | awk '{print $2}')
+    if [ $MANAGERCHECK == 'so-import' ]; then
+        TRUSTED_CONTAINERS=( \
+        "so-idstools" \
+        "so-nginx" \
+        "so-filebeat" \
+        "so-suricata" \
+        "so-soc" \
+        "so-elasticsearch" \
+        "so-kibana" \
+        "so-kratos" \
+        "so-suricata" \
+        "so-registry" \
+        "so-pcaptools" \
+        "so-zeek" )
+    elif [ $MANAGERCHECK != 'so-helix' ]; then
+        TRUSTED_CONTAINERS=( \
+        "so-acng" \
+        "so-thehive-cortex" \
+        "so-curator" \
+        "so-domainstats" \
+        "so-elastalert" \
+        "so-elasticsearch" \
+        "so-filebeat" \
+        "so-fleet" \
+        "so-fleet-launcher" \
+        "so-freqserver" \
+        "so-grafana" \
+        "so-idstools" \
+        "so-influxdb" \
+        "so-kibana" \
+        "so-kratos" \
+        "so-logstash" \
+        "so-minio" \
+        "so-mysql" \
+        "so-nginx" \
+        "so-pcaptools" \
+        "so-playbook" \
+        "so-redis" \
+        "so-soc" \
+        "so-soctopus" \
+        "so-steno" \
+        "so-strelka-frontend" \
+        "so-strelka-manager" \
+        "so-strelka-backend" \
+        "so-strelka-filestream" \
+        "so-suricata" \
+        "so-telegraf" \
+        "so-thehive" \
+        "so-thehive-es" \
+        "so-wazuh" \
+        "so-zeek" )
+    else
+        TRUSTED_CONTAINERS=( \
+        "so-filebeat" \
+        "so-idstools" \
+        "so-logstash" \
+        "so-nginx" \
+        "so-redis" \
+        "so-steno" \
+        "so-suricata" \
+        "so-telegraf" \
+        "so-zeek" )
+    fi
+}
+
+update_docker_containers() {
+  # Let's make sure we have the public key
+  curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS | gpg --import -
+  
+  CONTAINER_REGISTRY=quay.io
+  SIGNPATH=/root/sosigs
+  rm -rf $SIGNPATH
+  mkdir -p $SIGNPATH
+  if [ -z "$BRANCH" ]; then
+    BRANCH="master"
+  fi
+  # Download the containers from the interwebs
+  for i in "${TRUSTED_CONTAINERS[@]}"
+  do
+    # Pull down the trusted docker image
+    echo "Downloading $i"
+    docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION
+    
+    # Get signature
+    curl https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/$BRANCH/sigs/images/$VERSION/$i.sig --output $SIGNPATH/$i.sig
+    if [[ $? -ne 0 ]]; then
+      echo "Unable to pull signature file for $i:$VERSION"
+      exit 1
+    fi
+    # Dump our hash values
+    docker inspect $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION | jq '.[0].Created, .[0].RepoDigests, .[0].RootFS.Layers' > $SIGNPATH/$i.txt
+    if [[ $? -ne 0 ]]; then
+      echo "Unable to inspect $i:$VERSION"
+      exit 1
+    fi
+    GPGTEST=$(gpg --verify $SIGNPATH/$i.sig $SIGNPATH/$i.txt 2>&1)
+    if [[ $? -eq 0 ]]; then
+      # Tag it with the new registry destination
+      docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION $HOSTNAME:5000/$IMAGEREPO/$i:$VERSION
+      docker push $HOSTNAME:5000/$IMAGEREPO/$i:$VERSION
+    else
+      echo "There is a problem downloading the $i:$VERSION image. Details: "
+      echo ""
+      echo $GPGTEST
+      exit 1
+    fi
+  done
+  
+}

salt/common/tools/sbin/soup

@@ -16,6 +16,7 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 . /usr/sbin/so-common
+. /usr/sbin/so-image-common
 UPDATE_DIR=/tmp/sogh/securityonion
 INSTALLEDVERSION=$(cat /etc/soversion)
 INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk {'print $2'})
@@ -79,6 +80,24 @@ airgap_mounted() {
   fi
 }
 
+airgap_update_dockers() {
+  if [ $is_airgap -eq 0 ]; then
+    # Let's copy the tarball
+    if [ ! -f $AGDOCKER/registry.tar ]; then
+      echo "Unable to locate registry. Exiting"
+      exit 1
+    else
+      echo "Stopping the registry docker"
+      docker stop so-dockerregistry
+      docker rm so-dockerregistry
+      echo "Copying the new dockers over"
+      tar xvf $AGDOCKER/registry.tar -C /nsm/docker-registry/docker
+      echo "Add Registry back"
+      docker load -i $AGDOCKER/registry_image.tar
+    fi
+
+}
+
 check_airgap() {
   # See if this is an airgap install
   AIRGAP=$(cat /opt/so/saltstack/local/pillar/global.sls | grep airgap | awk '{print $2}')
@@ -290,103 +309,6 @@ update_centos_repo() {
   createrepo /nsm/repo
 }
 
-update_dockers() {
-  if [ $is_airgap -eq 0 ]; then
-    # Let's copy the tarball
-    if [ ! -f $AGDOCKER/registry.tar ]; then
-      echo "Unable to locate registry. Exiting"
-      exit 0
-    else
-      echo "Stopping the registry docker"
-      docker stop so-dockerregistry
-      docker rm so-dockerregistry
-      echo "Copying the new dockers over"
-      tar xvf $AGDOCKER/registry.tar -C /nsm/docker-registry/docker
-    fi
-  else
-    # List all the containers
-    if [ $MANAGERCHECK == 'so-import' ]; then
-      TRUSTED_CONTAINERS=( \
-      "so-idstools" \
-      "so-nginx" \
-      "so-filebeat" \
-      "so-suricata" \
-      "so-soc" \
-      "so-elasticsearch" \
-      "so-kibana" \
-      "so-kratos" \
-      "so-suricata" \
-      "so-registry" \
-      "so-pcaptools" \
-      "so-zeek" )
-    elif [ $MANAGERCHECK != 'so-helix' ]; then
-      TRUSTED_CONTAINERS=( \
-      "so-acng" \
-      "so-thehive-cortex" \
-      "so-curator" \
-      "so-domainstats" \
-      "so-elastalert" \
-      "so-elasticsearch" \
-      "so-filebeat" \
-      "so-fleet" \
-      "so-fleet-launcher" \
-      "so-freqserver" \
-      "so-grafana" \
-      "so-idstools" \
-      "so-influxdb" \
-      "so-kibana" \
-      "so-kratos" \
-      "so-logstash" \
-      "so-minio" \
-      "so-mysql" \
-      "so-nginx" \
-      "so-pcaptools" \
-      "so-playbook" \
-      "so-redis" \
-      "so-soc" \
-      "so-soctopus" \
-      "so-steno" \
-      "so-strelka-frontend" \
-      "so-strelka-manager" \
-      "so-strelka-backend" \
-      "so-strelka-filestream" \
-      "so-suricata" \
-      "so-telegraf" \
-      "so-thehive" \
-      "so-thehive-es" \
-      "so-wazuh" \
-      "so-zeek" )
-    else
-      TRUSTED_CONTAINERS=( \
-      "so-filebeat" \
-      "so-idstools" \
-      "so-logstash" \
-      "so-nginx" \
-      "so-redis" \
-      "so-steno" \
-      "so-suricata" \
-      "so-telegraf" \
-      "so-zeek" )
-    fi
-    
-# Download the containers from the interwebs
-    for i in "${TRUSTED_CONTAINERS[@]}"
-    do
-      # Pull down the trusted docker image
-      echo "Downloading $i:$NEWVERSION"
-      docker pull --disable-content-trust=false docker.io/$IMAGEREPO/$i:$NEWVERSION
-      # Tag it with the new registry destination
-      docker tag $IMAGEREPO/$i:$NEWVERSION $HOSTNAME:5000/$IMAGEREPO/$i:$NEWVERSION
-      docker push $HOSTNAME:5000/$IMAGEREPO/$i:$NEWVERSION 
-    done
-  fi
-    echo "Add Registry back if airgap"
-    if [ $is_airgap -eq 0 ]; then
-      docker load -i $AGDOCKER/registry_image.tar
-    fi 
-
-}
-
 update_version() {
   # Update the version to the latest
   echo "Updating the Security Onion version file."
@@ -513,7 +435,12 @@ echo ""
 echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION."
 echo ""
 echo "Updating dockers to $NEWVERSION."
-update_dockers
+if [ $is_airgap -eq 0 ]; then
+  airgap_update_dockers
+else
+  container_list
+  update_docker_containers
+fi
 echo ""
 echo "Stopping Salt Minion service."
 systemctl stop salt-minion

setup/so-functions

@@ -19,6 +19,8 @@ source ./so-whiptail
 source ./so-variables
 source ./so-common-functions
 
+CONTAINER_REGISTRY=quay.io
+
 SOVERSION=$(cat ../VERSION)
 
 log() {
@@ -874,79 +876,105 @@ docker_seed_registry() {
 	if ! [ -f /nsm/docker-registry/docker/registry.tar ]; then
 	    if [ "$install_type" == 'IMPORT' ]; then
 		local TRUSTED_CONTAINERS=(\
-			"so-idstools:$VERSION" \
+			"so-idstools" \
-			"so-nginx:$VERSION" \
+			"so-nginx" \
-			"so-filebeat:$VERSION" \
+			"so-filebeat" \
-			"so-suricata:$VERSION" \
+			"so-suricata" \
-			"so-soc:$VERSION" \
+			"so-soc" \
-			"so-steno:$VERSION" \
+			"so-steno" \
-			"so-elasticsearch:$VERSION" \
+			"so-elasticsearch" \
-			"so-kibana:$VERSION" \
+			"so-kibana" \
-			"so-kratos:$VERSION" \
+			"so-kratos" \
-			"so-suricata:$VERSION" \
+			"so-suricata" \
-			"so-pcaptools:$VERSION" \
+			"so-pcaptools" \
-			"so-zeek:$VERSION"
+			"so-zeek"
 		)
 		else
 		local TRUSTED_CONTAINERS=(\
-			"so-nginx:$VERSION" \
+			"so-nginx" \
-			"so-filebeat:$VERSION" \
+			"so-filebeat" \
-			"so-logstash:$VERSION" \
+			"so-logstash" \
-			"so-idstools:$VERSION" \
+			"so-idstools" \
-			"so-redis:$VERSION" \
+			"so-redis" \
-			"so-steno:$VERSION" \
+			"so-steno" \
-			"so-suricata:$VERSION" \
+			"so-suricata" \
-			"so-telegraf:$VERSION" \
+			"so-telegraf" \
-			"so-zeek:$VERSION"
+			"so-zeek"
 		)
 		fi
 		if [ "$install_type" != 'HELIXSENSOR' ] && [ "$install_type" != 'IMPORT' ]; then
 			TRUSTED_CONTAINERS=("${TRUSTED_CONTAINERS[@]}" \
-				"so-acng:$VERSION" \
+				"so-acng" \
-				"so-thehive-cortex:$VERSION" \
+				"so-thehive-cortex" \
-				"so-curator:$VERSION" \
+				"so-curator" \
-				"so-domainstats:$VERSION" \
+				"so-domainstats" \
-				"so-elastalert:$VERSION" \
+				"so-elastalert" \
-				"so-elasticsearch:$VERSION" \
+				"so-elasticsearch" \
-				"so-fleet:$VERSION" \
+				"so-fleet" \
-				"so-fleet-launcher:$VERSION" \
+				"so-fleet-launcher" \
-				"so-freqserver:$VERSION" \
+				"so-freqserver" \
-				"so-grafana:$VERSION" \
+				"so-grafana" \
-				"so-influxdb:$VERSION" \
+				"so-influxdb" \
-				"so-kibana:$VERSION" \
+				"so-kibana" \
-				"so-minio:$VERSION" \
+				"so-minio" \
-				"so-mysql:$VERSION" \
+				"so-mysql" \
-				"so-pcaptools:$VERSION" \
+				"so-pcaptools" \
-				"so-playbook:$VERSION" \
+				"so-playbook" \
-				"so-soc:$VERSION" \
+				"so-soc" \
-				"so-kratos:$VERSION" \
+				"so-kratos" \
-				"so-soctopus:$VERSION" \
+				"so-soctopus" \
-				"so-steno:$VERSION" \
+				"so-steno" \
-				"so-strelka-frontend:$VERSION" \
+				"so-strelka-frontend" \
-				"so-strelka-manager:$VERSION" \
+				"so-strelka-manager" \
-				"so-strelka-backend:$VERSION" \
+				"so-strelka-backend" \
-				"so-strelka-filestream:$VERSION" \
+				"so-strelka-filestream" \
-				"so-thehive:$VERSION" \
+				"so-thehive" \
-				"so-thehive-es:$VERSION" \
+				"so-thehive-es" \
-				"so-wazuh:$VERSION"
+				"so-wazuh"
 			)
 		fi
 		local percent=25
+		# Let's make sure we have the public key
+        curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS | gpg --import -
+  
+        SIGNPATH=/root/sosigs
+        rm -rf $SIGNPATH
+        mkdir -p $SIGNPATH
+        if [ -z "$BRANCH" ]; then
+            BRANCH="master"
+        fi
 		for i in "${TRUSTED_CONTAINERS[@]}"; do
 			if [ "$install_type" != 'HELIXSENSOR' ]; then ((percent=percent+1)); else ((percent=percent+6)); fi
 			# Pull down the trusted docker image
-			set_progress_str "$percent" "Downloading $i"
+			set_progress_str "$percent" "Downloading $i:$VERSION"
 			{
+				echo "Downloading $i"
+                docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION
     
-				if ! docker pull --disable-content-trust=false docker.io/$IMAGEREPO/"$i"; then
+    			# Get signature
-					sleep 5
+    			curl https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/$BRANCH/sigs/images/$VERSION/$i.sig --output $SIGNPATH/$i.sig
-					docker pull --disable-content-trust=false docker.io/$IMAGEREPO/"$i"
+    			if [[ $? -ne 0 ]]; then
-				fi
+      				echo "Unable to pull signature file for $i:$VERSION"
-				# Tag it with the new registry destination
+      				exit 1
-				docker tag $IMAGEREPO/"$i" "$HOSTNAME":5000/$IMAGEREPO/"$i"
+    			fi
-				docker push "$HOSTNAME":5000/$IMAGEREPO/"$i"
+    			# Dump our hash values
-				#docker rmi $IMAGEREPO/"$i"
+    			docker inspect $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION | jq '.[0].Created, .[0].RepoDigests, .[0].RootFS.Layers' > $SIGNPATH/$i.txt
-			} >> "$setup_log" 2>&1
+    			if [[ $? -ne 0 ]]; then
+      				echo "Unable to inspect $i"
+      				exit 1
+    			fi
+    				GPGTEST=$(gpg --verify $SIGNPATH/$i.sig $SIGNPATH/$i.txt 2>&1)
+    			if [[ $? -eq 0 ]]; then
+      			# Tag it with the new registry destination
+      				docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION $HOSTNAME:5000/$IMAGEREPO/$i:$VERSION
+      				docker push $HOSTNAME:5000/$IMAGEREPO/$i:$VERSION
+    			else
+      				echo "There is a problem downloading the $i image. Details: "
+      				echo ""
+      				echo $GPGTEST
+      				exit 1
+    			fi
+ 			} >> "$setup_log" 2>&1
 		done
 	else
 		tar xvf /nsm/docker-registry/docker/registry.tar -C /nsm/docker-registry/docker >> "$setup_log" 2>&1