Merge pull request #1942 from Security-Onion-Solutions/jertel/refactor-seed

Jertel/refactor seed
2026-01-23 08:31:30 +01:00 · 2020-11-16 18:46:28 -05:00
parent a8644478b5 4311f66110
commit 5c310327e4
8 changed files with 89 additions and 154 deletions
--- a/salt/common/tools/sbin/so-image-common
+++ b/salt/common/tools/sbin/so-image-common
@@ -19,29 +19,30 @@
 IMAGEREPO=securityonion

 container_list() {
-    MANAGERCHECK=so-unknown
-    if [ -f /etc/salt/grains ]; then
-      MANAGERCHECK=$(cat /etc/salt/grains | grep role | awk '{print $2}')
+    MANAGERCHECK=$1
+    if [ -z "$MANAGERCHECK" ]; then
+      MANAGERCHECK=so-unknown
+      if [ -f /etc/salt/grains ]; then
+        MANAGERCHECK=$(cat /etc/salt/grains | grep role | awk '{print $2}')
+      fi
    fi

    if [ $MANAGERCHECK == 'so-import' ]; then
-        TRUSTED_CONTAINERS=( \
-        "so-idstools" \
-        "so-nginx" \
-        "so-filebeat" \
-        "so-suricata" \
-        "so-soc" \
+      TRUSTED_CONTAINERS=( \
        "so-elasticsearch" \
+        "so-filebeat" \
+        "so-idstools" \
        "so-kibana" \
        "so-kratos" \
-        "so-suricata" \
-        "so-registry" \
+        "so-nginx" \
        "so-pcaptools" \
+        "so-soc" \
+        "so-steno" \
+        "so-suricata" \
        "so-zeek" )
    elif [ $MANAGERCHECK != 'so-helix' ]; then
-        TRUSTED_CONTAINERS=( \
+      TRUSTED_CONTAINERS=( \
        "so-acng" \
-        "so-thehive-cortex" \
        "so-curator" \
        "so-domainstats" \
        "so-elastalert" \
@@ -65,18 +66,19 @@ container_list() {
        "so-soc" \
        "so-soctopus" \
        "so-steno" \
-        "so-strelka-frontend" \
-        "so-strelka-manager" \
        "so-strelka-backend" \
        "so-strelka-filestream" \
+        "so-strelka-frontend" \
+        "so-strelka-manager" \
        "so-suricata" \
        "so-telegraf" \
        "so-thehive" \
+        "so-thehive-cortex" \
        "so-thehive-es" \
        "so-wazuh" \
        "so-zeek" )
    else
-        TRUSTED_CONTAINERS=( \
+      TRUSTED_CONTAINERS=( \
        "so-filebeat" \
        "so-idstools" \
        "so-logstash" \
@@ -90,16 +92,22 @@ container_list() {
 }

 update_docker_containers() {
-  CURLTYPE=$1
-  IMAGE_TAG_SUFFIX=$2
+  local CURLTYPE=$1
+  local IMAGE_TAG_SUFFIX=$2
+  local PROGRESS_CALLBACK=$3
+  local LOG_FILE=$4

-  CONTAINER_REGISTRY=quay.io
-  SIGNPATH=/root/sosigs
+  local CONTAINER_REGISTRY=quay.io
+  local SIGNPATH=/root/sosigs
  
  if [ -z "$CURLTYPE" ]; then
    CURLTYPE=unknown
  fi

+  if [ -z "$LOG_FILE" ]; then
+    LOG_FILE=/dev/tty
+  fi
+
  # Recheck the version for scenarios were the VERSION wasn't known before this script was imported
  set_version
  set_os
@@ -109,50 +117,55 @@ update_docker_containers() {
  fi

  # Let's make sure we have the public key
-  curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS | gpg --import -
+  curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS | gpg --import -  >> "$LOG_FILE" 2>&1
  
-  rm -rf $SIGNPATH
-  mkdir -p $SIGNPATH
+  rm -rf $SIGNPATH >> "$LOG_FILE" 2>&1 
+  mkdir -p $SIGNPATH >> "$LOG_FILE" 2>&1 

  # Download the containers from the interwebs
  for i in "${TRUSTED_CONTAINERS[@]}"
  do
+    if [ -z "$PROGRESS_CALLBACK" ]; then
+      echo "Downloading $i" >> "$LOG_FILE" 2>&1 
+    else
+      $PROGRESS_CALLBACK $i
+    fi
+
    # Pull down the trusted docker image
-    echo "Downloading $i"
-    docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION$IMAGE_TAG_SUFFIX
+    local image=$i:$VERSION$IMAGE_TAG_SUFFIX
+    docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 
    
    # Get signature
-    curl -A "$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)" https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$i:$VERSION$IMAGE_TAG_SUFFIX.sig
+    curl -A "$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)" https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig >> "$LOG_FILE" 2>&1 
    if [[ $? -ne 0 ]]; then
-      echo "Unable to pull signature file for $i:$VERSION$IMAGE_TAG_SUFFIX"
+      echo "Unable to pull signature file for $image" >> "$LOG_FILE" 2>&1 
      exit 1
    fi
    # Dump our hash values
-    DOCKERINSPECT=$(docker inspect $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION$IMAGE_TAG_SUFFIX)
+    DOCKERINSPECT=$(docker inspect $CONTAINER_REGISTRY/$IMAGEREPO/$image)
       
-    echo "$DOCKERINSPECT" | jq ".[0].RepoDigests[] | select(. | contains(\"$CONTAINER_REGISTRY\"))" > $SIGNPATH/$i:$VERSION$IMAGE_TAG_SUFFIX.txt
-    echo "$DOCKERINSPECT" | jq ".[0].Created, .[0].RootFS.Layers" >> $SIGNPATH/$i:$VERSION$IMAGE_TAG_SUFFIX.txt
+    echo "$DOCKERINSPECT" | jq ".[0].RepoDigests[] | select(. | contains(\"$CONTAINER_REGISTRY\"))" > $SIGNPATH/$image.txt
+    echo "$DOCKERINSPECT" | jq ".[0].Created, .[0].RootFS.Layers" >> $SIGNPATH/$image.txt
        
    if [[ $? -ne 0 ]]; then
-      echo "Unable to inspect $i:$VERSION$IMAGE_TAG_SUFFIX"
+      echo "Unable to inspect $image" >> "$LOG_FILE" 2>&1 
      exit 1
    fi
-    GPGTEST=$(gpg --verify $SIGNPATH/$i:$VERSION$IMAGE_TAG_SUFFIX.sig $SIGNPATH/$i:$VERSION$IMAGE_TAG_SUFFIX.txt 2>&1)
+    GPGTEST=$(gpg --verify $SIGNPATH/$image.sig $SIGNPATH/$image.txt 2>&1)
    if [[ $? -eq 0 ]]; then
      if [[ -z "$SKIP_TAGPUSH" ]]; then
        # Tag it with the new registry destination
        if [ -z "$HOSTNAME" ]; then
          HOSTNAME=$(hostname)
        fi
-        docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION$IMAGE_TAG_SUFFIX $HOSTNAME:5000/$IMAGEREPO/$i:$VERSION$IMAGE_TAG_SUFFIX
-        docker push $HOSTNAME:5000/$IMAGEREPO/$i:$VERSION$IMAGE_TAG_SUFFIX
+        docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$image $HOSTNAME:5000/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 
+        docker push $HOSTNAME:5000/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 
      fi
    else
-      echo "There is a problem downloading the $i:$VERSION$IMAGE_TAG_SUFFIX image. Details: "
-      echo ""
-      echo $GPGTEST
+      echo "There is a problem downloading the $image image. Details: " >> "$LOG_FILE" 2>&1 
+      echo "" >> "$LOG_FILE" 2>&1 
+      echo $GPGTEST >> "$LOG_FILE" 2>&1 
      exit 1
    fi
  done
-  
 }
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -93,7 +93,12 @@ airgap_update_dockers() {
      docker load -i $AGDOCKER/registry_image.tar
    fi
  fi
+}

+update_registry() {
+  docker stop so-dockerregistry
+  docker rm so-dockerregistry
+  salt-call state.apply registry
 }

 check_airgap() {
@@ -431,6 +436,7 @@ echo "Updating dockers to $NEWVERSION."
 if [ $is_airgap -eq 0 ]; then
  airgap_update_dockers
 else
+  update_registry
  update_docker_containers "soup"
 fi
 echo ""
--- a/salt/domainstats/init.sls
+++ b/salt/domainstats/init.sls
@@ -43,13 +43,13 @@ dstatslogdir:

 so-domainstatsimage:
 cmd.run:
-   - name: docker pull --disable-content-trust=false docker.io/{{ IMAGEREPO }}/so-domainstats:HH1.0.3
+   - name: docker pull {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-domainstats:{{ VERSION }}

 so-domainstats:
  docker_container.running:
    - require:
      - so-domainstatsimage
-    - image: docker.io/{{ IMAGEREPO }}/so-domainstats:HH1.0.3
+    - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-domainstats:{{ VERSION }}
    - hostname: domainstats
    - name: so-domainstats
    - user: domainstats
--- a/salt/freqserver/init.sls
+++ b/salt/freqserver/init.sls
@@ -43,13 +43,13 @@ freqlogdir:

 so-freqimage:
 cmd.run:
-   - name: docker pull --disable-content-trust=false docker.io/{{ IMAGEREPO }}/so-freqserver:HH1.0.3
+   - name: docker pull {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-freqserver:{{ VERSION }}

 so-freq:
  docker_container.running:
    - require:
      - so-freqimage
-    - image: docker.io/{{ IMAGEREPO }}/so-freqserver:HH1.0.3
+    - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-freqserver:{{ VERSION }}
    - hostname: freqserver
    - name: so-freqserver
    - user: freqserver
--- a/salt/nodered/init.sls
+++ b/salt/nodered/init.sls
@@ -67,7 +67,7 @@ noderedlog:

 so-nodered:
  docker_container.running:
-    - image: {{ IMAGEREPO }}/so-nodered:HH1.2.2
+    - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-nodered:{{ VERSION }}
    - interactive: True
    - binds:
      - /opt/so/conf/nodered/:/data:rw
--- a/salt/registry/init.sls
+++ b/salt/registry/init.sls
@@ -45,7 +45,7 @@ dockerregistryconf:
 # Install the registry container
 so-dockerregistry:
  docker_container.running:
-    - image: registry:latest
+    - image: ghcr.io/security-onion-solutions/registry:latest
    - hostname: so-registry
    - restart_policy: always
    - port_bindings:
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -743,7 +743,7 @@ detect_os() {
 				systemctl start NetworkManager;
 			} >> "$setup_log" 2<&1
 		fi
-		apt-get install -y bc >> "$setup_log" 2>&1
+		apt-get install -y bc curl >> "$setup_log" 2>&1

 	else
 		echo "We were unable to determine if you are using a supported OS."
@@ -870,116 +870,32 @@ docker_registry() {

 }

+docker_seed_update() {
+	local name=$1
+	local percent_delta=1
+	if [ "$install_type" == 'HELIXSENSOR' ]; then 
+		percent_delta=6
+	fi
+	((docker_seed_update_percent=docker_seed_update_percent+percent_delta))
+
+	set_progress_str "$docker_seed_update_percent" "Downloading $name"
+}
+
 docker_seed_registry() {
 	local VERSION="$SOVERSION"

 	if ! [ -f /nsm/docker-registry/docker/registry.tar ]; then
-	    if [ "$install_type" == 'IMPORT' ]; then
-		local TRUSTED_CONTAINERS=(\
-			"so-idstools" \
-			"so-nginx" \
-			"so-filebeat" \
-			"so-suricata" \
-			"so-soc" \
-			"so-steno" \
-			"so-elasticsearch" \
-			"so-kibana" \
-			"so-kratos" \
-			"so-suricata" \
-			"so-pcaptools" \
-			"so-zeek"
-		)
+		if [ "$install_type" == 'IMPORT' ]; then
+			container_list 'so-import'
+		elif [ "$install_type" == 'HELIXSENSOR' ]; then
+			container_list 'so-helix'
 		else
-		local TRUSTED_CONTAINERS=(\
-			"so-nginx" \
-			"so-filebeat" \
-			"so-logstash" \
-			"so-idstools" \
-			"so-redis" \
-			"so-steno" \
-			"so-suricata" \
-			"so-telegraf" \
-			"so-zeek"
-		)
+			container_list
 		fi
-		if [ "$install_type" != 'HELIXSENSOR' ] && [ "$install_type" != 'IMPORT' ]; then
-			TRUSTED_CONTAINERS=("${TRUSTED_CONTAINERS[@]}" \
-				"so-acng" \
-				"so-thehive-cortex" \
-				"so-curator" \
-				"so-domainstats" \
-				"so-elastalert" \
-				"so-elasticsearch" \
-				"so-fleet" \
-				"so-fleet-launcher" \
-				"so-freqserver" \
-				"so-grafana" \
-				"so-influxdb" \
-				"so-kibana" \
-				"so-minio" \
-				"so-mysql" \
-				"so-pcaptools" \
-				"so-playbook" \
-				"so-soc" \
-				"so-kratos" \
-				"so-soctopus" \
-				"so-steno" \
-				"so-strelka-frontend" \
-				"so-strelka-manager" \
-				"so-strelka-backend" \
-				"so-strelka-filestream" \
-				"so-thehive" \
-				"so-thehive-es" \
-				"so-wazuh"
-			)
-		fi
-		local percent=25
-		# Let's make sure we have the public key
-        curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS | gpg --import -
-  
-        SIGNPATH=/root/sosigs
-        rm -rf $SIGNPATH
-        mkdir -p $SIGNPATH
-        if [ -z "$BRANCH" ]; then
-            BRANCH="master"
-        fi
-		for i in "${TRUSTED_CONTAINERS[@]}"; do
-			if [ "$install_type" != 'HELIXSENSOR' ]; then ((percent=percent+1)); else ((percent=percent+6)); fi
-			# Pull down the trusted docker image
-			set_progress_str "$percent" "Downloading $i:$VERSION"
-			{
-				echo "Downloading $i"
-                        docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION
-    
-    			# Get signature
-    			curl -A "netinstall/$OS/$(uname -r)" https://sigs.securityonion.net/$VERSION/$i:$VERSION.sig --output $SIGNPATH/$i:$VERSION.sig
-    			if [[ $? -ne 0 ]]; then
-      				echo "Unable to pull signature file for $i:$VERSION"
-      				exit 1
-    			fi
-    			# Dump our hash values
-			    DOCKERINSPECT=$(docker inspect $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION)
-    
-                echo "$DOCKERINSPECT" | jq ".[0].RepoDigests[] | select(. | contains(\"$CONTAINER_REGISTRY\"))" > $SIGNPATH/$i:$VERSION.txt
-                echo "$DOCKERINSPECT" | jq ".[0].Created, .[0].RootFS.Layers" >> $SIGNPATH/$i:$VERSION.txt
-			
-			    if [[ $? -ne 0 ]]; then
-      				echo "Unable to inspect $i"
-      				exit 1
-    			fi
-    				GPGTEST=$(gpg --verify $SIGNPATH/$i:$VERSION.sig $SIGNPATH/$i:$VERSION.txt 2>&1)
-    			if [[ $? -eq 0 ]]; then
-      			# Tag it with the new registry destination
-      				docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION $HOSTNAME:5000/$IMAGEREPO/$i:$VERSION
-      				docker push $HOSTNAME:5000/$IMAGEREPO/$i:$VERSION
-    			else
-      				echo "There is a problem downloading the $i image. Details: "
-      				echo ""
-      				echo $GPGTEST
-      				exit 1
-    			fi
- 			} >> "$setup_log" 2>&1
-		done
+
+		docker_seed_update_percent=25
+
+		update_docker_containers 'netinstall' '' 'docker_seed_update' "$setup_log"
 	else
 		tar xvf /nsm/docker-registry/docker/registry.tar -C /nsm/docker-registry/docker >> "$setup_log" 2>&1
 		rm /nsm/docker-registry/docker/registry.tar >> "$setup_log" 2>&1
@@ -1006,10 +922,10 @@ firewall_generate_templates() {
 	local firewall_pillar_path=$local_salt_dir/salt/firewall
 	mkdir -p "$firewall_pillar_path"
   
-    cp ../files/firewall/* /opt/so/saltstack/local/salt/firewall/ >> "$setup_log" 2>&1
+	cp ../files/firewall/* /opt/so/saltstack/local/salt/firewall/ >> "$setup_log" 2>&1

-    for i in analyst beats_endpoint sensor manager minion osquery_endpoint search_node wazuh_endpoint; do
-	  $default_salt_dir/salt/common/tools/sbin/so-firewall includehost "$i" 127.0.0.1
+	for i in analyst beats_endpoint sensor manager minion osquery_endpoint search_node wazuh_endpoint; do
+		$default_salt_dir/salt/common/tools/sbin/so-firewall includehost "$i" 127.0.0.1
 	done

 }
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -599,9 +599,9 @@ fi
 		else
 			set_progress_str 26 'Downloading containers from the internet'
 		fi
-        import_registry_docker >> $setup_log 2>&1
+		import_registry_docker >> $setup_log 2>&1
 		salt-call state.apply -l info registry >> $setup_log 2>&1
-		docker_seed_registry  2>> "$setup_log" # ~ 60% when finished
+		docker_seed_registry # ~ 60% when finished
 		
 		set_progress_str 60 "$(print_salt_state_apply 'manager')"
 		if [[ "$STRELKARULES" == 1 ]]; then