From 2ff738a61cd5c6ed0a91bce8ddd100dbb5c54b01 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 16 Nov 2020 13:27:23 -0500 Subject: [PATCH 1/8] Refactor docker_seed_registry to eliminate duplicate logic --- salt/common/tools/sbin/so-image-common | 73 +++++++------ setup/so-functions | 140 ++++++------------------- 2 files changed, 74 insertions(+), 139 deletions(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index a4eeb5239..4a3a099bc 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -19,29 +19,30 @@ IMAGEREPO=securityonion container_list() { - MANAGERCHECK=so-unknown - if [ -f /etc/salt/grains ]; then - MANAGERCHECK=$(cat /etc/salt/grains | grep role | awk '{print $2}') + MANAGERCHECK=$1 + if [ -z "$MANAGERCHECK" ]; then + MANAGERCHECK=so-unknown + if [ -f /etc/salt/grains ]; then + MANAGERCHECK=$(cat /etc/salt/grains | grep role | awk '{print $2}') + fi fi if [ $MANAGERCHECK == 'so-import' ]; then - TRUSTED_CONTAINERS=( \ - "so-idstools" \ - "so-nginx" \ - "so-filebeat" \ - "so-suricata" \ - "so-soc" \ + TRUSTED_CONTAINERS=( \ "so-elasticsearch" \ + "so-filebeat" \ + "so-idstools" \ "so-kibana" \ "so-kratos" \ - "so-suricata" \ - "so-registry" \ + "so-nginx" \ "so-pcaptools" \ + "so-soc" \ + "so-steno" \ + "so-suricata" \ "so-zeek" ) elif [ $MANAGERCHECK != 'so-helix' ]; then - TRUSTED_CONTAINERS=( \ + TRUSTED_CONTAINERS=( \ "so-acng" \ - "so-thehive-cortex" \ "so-curator" \ "so-domainstats" \ "so-elastalert" \ @@ -65,18 +66,19 @@ container_list() { "so-soc" \ "so-soctopus" \ "so-steno" \ - "so-strelka-frontend" \ - "so-strelka-manager" \ "so-strelka-backend" \ "so-strelka-filestream" \ + "so-strelka-frontend" \ + "so-strelka-manager" \ "so-suricata" \ "so-telegraf" \ "so-thehive" \ + "so-thehive-cortex" \ "so-thehive-es" \ "so-wazuh" \ "so-zeek" ) else - TRUSTED_CONTAINERS=( \ + TRUSTED_CONTAINERS=( \ "so-filebeat" \ "so-idstools" \ "so-logstash" \ @@ -90,11 +92,12 @@ container_list() { } update_docker_containers() { - CURLTYPE=$1 - IMAGE_TAG_SUFFIX=$2 + local CURLTYPE=$1 + local IMAGE_TAG_SUFFIX=$2 + local PROGRESS_CALLBACK=$3 - CONTAINER_REGISTRY=quay.io - SIGNPATH=/root/sosigs + local CONTAINER_REGISTRY=quay.io + local SIGNPATH=/root/sosigs if [ -z "$CURLTYPE" ]; then CURLTYPE=unknown @@ -117,38 +120,44 @@ update_docker_containers() { # Download the containers from the interwebs for i in "${TRUSTED_CONTAINERS[@]}" do + if [ -z "$PROGRESS_CALLBACK" ]; then + echo "Downloading $i" + else + $PROGRESS_CALLBACK $i + fi + # Pull down the trusted docker image - echo "Downloading $i" - docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION$IMAGE_TAG_SUFFIX + local image=$i:$VERSION$IMAGE_TAG_SUFFIX + docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image # Get signature - curl -A "$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)" https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$i:$VERSION$IMAGE_TAG_SUFFIX.sig + curl -A "$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)" https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig if [[ $? -ne 0 ]]; then - echo "Unable to pull signature file for $i:$VERSION$IMAGE_TAG_SUFFIX" + echo "Unable to pull signature file for $image" exit 1 fi # Dump our hash values - DOCKERINSPECT=$(docker inspect $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION$IMAGE_TAG_SUFFIX) + DOCKERINSPECT=$(docker inspect $CONTAINER_REGISTRY/$IMAGEREPO/$image) - echo "$DOCKERINSPECT" | jq ".[0].RepoDigests[] | select(. | contains(\"$CONTAINER_REGISTRY\"))" > $SIGNPATH/$i:$VERSION$IMAGE_TAG_SUFFIX.txt - echo "$DOCKERINSPECT" | jq ".[0].Created, .[0].RootFS.Layers" >> $SIGNPATH/$i:$VERSION$IMAGE_TAG_SUFFIX.txt + echo "$DOCKERINSPECT" | jq ".[0].RepoDigests[] | select(. | contains(\"$CONTAINER_REGISTRY\"))" > $SIGNPATH/$image.txt + echo "$DOCKERINSPECT" | jq ".[0].Created, .[0].RootFS.Layers" >> $SIGNPATH/$image.txt if [[ $? -ne 0 ]]; then - echo "Unable to inspect $i:$VERSION$IMAGE_TAG_SUFFIX" + echo "Unable to inspect $image" exit 1 fi - GPGTEST=$(gpg --verify $SIGNPATH/$i:$VERSION$IMAGE_TAG_SUFFIX.sig $SIGNPATH/$i:$VERSION$IMAGE_TAG_SUFFIX.txt 2>&1) + GPGTEST=$(gpg --verify $SIGNPATH/$image.sig $SIGNPATH/$image.txt 2>&1) if [[ $? -eq 0 ]]; then if [[ -z "$SKIP_TAGPUSH" ]]; then # Tag it with the new registry destination if [ -z "$HOSTNAME" ]; then HOSTNAME=$(hostname) fi - docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION$IMAGE_TAG_SUFFIX $HOSTNAME:5000/$IMAGEREPO/$i:$VERSION$IMAGE_TAG_SUFFIX - docker push $HOSTNAME:5000/$IMAGEREPO/$i:$VERSION$IMAGE_TAG_SUFFIX + docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$image $HOSTNAME:5000/$IMAGEREPO/$image + docker push $HOSTNAME:5000/$IMAGEREPO/$image fi else - echo "There is a problem downloading the $i:$VERSION$IMAGE_TAG_SUFFIX image. Details: " + echo "There is a problem downloading the $image image. Details: " echo "" echo $GPGTEST exit 1 diff --git a/setup/so-functions b/setup/so-functions index 642ae5004..3afc97b6c 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -23,6 +23,11 @@ CONTAINER_REGISTRY=quay.io SOVERSION=$(cat ../VERSION) +# Duplicate stdout and stderr file descriptors for use with whiptail +# Using >&10 or >&20 will override any ancestral >> or > redirects and send +# to stdout or stderr, repsectively. +exec 10>&1 20>&2 + log() { msg=$1 level=${2:-I} @@ -870,116 +875,37 @@ docker_registry() { } +docker_seed_update() { + local name=$1 + local percent_delta=1 + if [ "$install_type" == 'HELIXSENSOR' ]; then + percent_delta=6 + fi + ((docker_seed_update_percent=docker_seed_update_percent+percent_delta)) + + # Backup current output descriptors and reset to normal + exec 8>&1 9>&2 1>&10 2>&20 + + set_progress_str "$docker_seed_update_percent" "Downloading $name" + + # Restore current output descriptors and remove backups + exec 1>&8- 2>&9- +} + docker_seed_registry() { local VERSION="$SOVERSION" if ! [ -f /nsm/docker-registry/docker/registry.tar ]; then - if [ "$install_type" == 'IMPORT' ]; then - local TRUSTED_CONTAINERS=(\ - "so-idstools" \ - "so-nginx" \ - "so-filebeat" \ - "so-suricata" \ - "so-soc" \ - "so-steno" \ - "so-elasticsearch" \ - "so-kibana" \ - "so-kratos" \ - "so-suricata" \ - "so-pcaptools" \ - "so-zeek" - ) + if [ "$install_type" == 'IMPORT' ]; then + container_list 'so-import' + elif [ "$install_type" != 'HELIXSENSOR' ]; then + container_list 'so-helix' else - local TRUSTED_CONTAINERS=(\ - "so-nginx" \ - "so-filebeat" \ - "so-logstash" \ - "so-idstools" \ - "so-redis" \ - "so-steno" \ - "so-suricata" \ - "so-telegraf" \ - "so-zeek" - ) + container_list fi - if [ "$install_type" != 'HELIXSENSOR' ] && [ "$install_type" != 'IMPORT' ]; then - TRUSTED_CONTAINERS=("${TRUSTED_CONTAINERS[@]}" \ - "so-acng" \ - "so-thehive-cortex" \ - "so-curator" \ - "so-domainstats" \ - "so-elastalert" \ - "so-elasticsearch" \ - "so-fleet" \ - "so-fleet-launcher" \ - "so-freqserver" \ - "so-grafana" \ - "so-influxdb" \ - "so-kibana" \ - "so-minio" \ - "so-mysql" \ - "so-pcaptools" \ - "so-playbook" \ - "so-soc" \ - "so-kratos" \ - "so-soctopus" \ - "so-steno" \ - "so-strelka-frontend" \ - "so-strelka-manager" \ - "so-strelka-backend" \ - "so-strelka-filestream" \ - "so-thehive" \ - "so-thehive-es" \ - "so-wazuh" - ) - fi - local percent=25 - # Let's make sure we have the public key - curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS | gpg --import - - - SIGNPATH=/root/sosigs - rm -rf $SIGNPATH - mkdir -p $SIGNPATH - if [ -z "$BRANCH" ]; then - BRANCH="master" - fi - for i in "${TRUSTED_CONTAINERS[@]}"; do - if [ "$install_type" != 'HELIXSENSOR' ]; then ((percent=percent+1)); else ((percent=percent+6)); fi - # Pull down the trusted docker image - set_progress_str "$percent" "Downloading $i:$VERSION" - { - echo "Downloading $i" - docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION - - # Get signature - curl -A "netinstall/$OS/$(uname -r)" https://sigs.securityonion.net/$VERSION/$i:$VERSION.sig --output $SIGNPATH/$i:$VERSION.sig - if [[ $? -ne 0 ]]; then - echo "Unable to pull signature file for $i:$VERSION" - exit 1 - fi - # Dump our hash values - DOCKERINSPECT=$(docker inspect $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION) - - echo "$DOCKERINSPECT" | jq ".[0].RepoDigests[] | select(. | contains(\"$CONTAINER_REGISTRY\"))" > $SIGNPATH/$i:$VERSION.txt - echo "$DOCKERINSPECT" | jq ".[0].Created, .[0].RootFS.Layers" >> $SIGNPATH/$i:$VERSION.txt - - if [[ $? -ne 0 ]]; then - echo "Unable to inspect $i" - exit 1 - fi - GPGTEST=$(gpg --verify $SIGNPATH/$i:$VERSION.sig $SIGNPATH/$i:$VERSION.txt 2>&1) - if [[ $? -eq 0 ]]; then - # Tag it with the new registry destination - docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION $HOSTNAME:5000/$IMAGEREPO/$i:$VERSION - docker push $HOSTNAME:5000/$IMAGEREPO/$i:$VERSION - else - echo "There is a problem downloading the $i image. Details: " - echo "" - echo $GPGTEST - exit 1 - fi - } >> "$setup_log" 2>&1 - done + + docker_seed_update_percent=25 + update_docker_containers 'netinstall' '' 'docker_seed_update' >> "$setup_log" 2>&1 else tar xvf /nsm/docker-registry/docker/registry.tar -C /nsm/docker-registry/docker >> "$setup_log" 2>&1 rm /nsm/docker-registry/docker/registry.tar >> "$setup_log" 2>&1 @@ -1006,10 +932,10 @@ firewall_generate_templates() { local firewall_pillar_path=$local_salt_dir/salt/firewall mkdir -p "$firewall_pillar_path" - cp ../files/firewall/* /opt/so/saltstack/local/salt/firewall/ >> "$setup_log" 2>&1 + cp ../files/firewall/* /opt/so/saltstack/local/salt/firewall/ >> "$setup_log" 2>&1 - for i in analyst beats_endpoint sensor manager minion osquery_endpoint search_node wazuh_endpoint; do - $default_salt_dir/salt/common/tools/sbin/so-firewall includehost "$i" 127.0.0.1 + for i in analyst beats_endpoint sensor manager minion osquery_endpoint search_node wazuh_endpoint; do + $default_salt_dir/salt/common/tools/sbin/so-firewall includehost "$i" 127.0.0.1 done } From a343e3f31ea0907d09bc5ad4da38b4e2a1cb3ba2 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 16 Nov 2020 14:10:48 -0500 Subject: [PATCH 2/8] Save descriptors while inside the progress pipe --- setup/so-functions | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 3afc97b6c..8ec78787e 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -23,11 +23,6 @@ CONTAINER_REGISTRY=quay.io SOVERSION=$(cat ../VERSION) -# Duplicate stdout and stderr file descriptors for use with whiptail -# Using >&10 or >&20 will override any ancestral >> or > redirects and send -# to stdout or stderr, repsectively. -exec 10>&1 20>&2 - log() { msg=$1 level=${2:-I} @@ -883,7 +878,7 @@ docker_seed_update() { fi ((docker_seed_update_percent=docker_seed_update_percent+percent_delta)) - # Backup current output descriptors and reset to normal + # Backup current output descriptors and reset to saved descriptors from docker_seed_registry function exec 8>&1 9>&2 1>&10 2>&20 set_progress_str "$docker_seed_update_percent" "Downloading $name" @@ -905,6 +900,10 @@ docker_seed_registry() { fi docker_seed_update_percent=25 + + # Save output descriptors for use in docker_seed_registry_update function + exec 10>&1 20>&2 + update_docker_containers 'netinstall' '' 'docker_seed_update' >> "$setup_log" 2>&1 else tar xvf /nsm/docker-registry/docker/registry.tar -C /nsm/docker-registry/docker >> "$setup_log" 2>&1 From 8234b6f83565aa448a5d985ccda35f22f42633ab Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 16 Nov 2020 15:11:08 -0500 Subject: [PATCH 3/8] Switch remaining containers over to new registries; Continued bash refactoring --- salt/common/tools/sbin/so-image-common | 30 +++++++++++++++----------- salt/domainstats/init.sls | 4 ++-- salt/freqserver/init.sls | 4 ++-- salt/nodered/init.sls | 2 +- salt/registry/init.sls | 2 +- setup/so-functions | 5 +---- setup/so-setup | 4 ++-- 7 files changed, 26 insertions(+), 25 deletions(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 4a3a099bc..aefeade91 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -95,6 +95,7 @@ update_docker_containers() { local CURLTYPE=$1 local IMAGE_TAG_SUFFIX=$2 local PROGRESS_CALLBACK=$3 + local LOG_FILE=$4 local CONTAINER_REGISTRY=quay.io local SIGNPATH=/root/sosigs @@ -103,6 +104,10 @@ update_docker_containers() { CURLTYPE=unknown fi + if [ -z "$LOG_FILE" ]; then + LOG_FILE=/dev/tty + fi + # Recheck the version for scenarios were the VERSION wasn't known before this script was imported set_version set_os @@ -114,26 +119,26 @@ update_docker_containers() { # Let's make sure we have the public key curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS | gpg --import - - rm -rf $SIGNPATH - mkdir -p $SIGNPATH + rm -rf $SIGNPATH >> "$LOG_FILE" 2>&1 + mkdir -p $SIGNPATH >> "$LOG_FILE" 2>&1 # Download the containers from the interwebs for i in "${TRUSTED_CONTAINERS[@]}" do if [ -z "$PROGRESS_CALLBACK" ]; then - echo "Downloading $i" + echo "Downloading $i" >> "$LOG_FILE" 2>&1 else $PROGRESS_CALLBACK $i fi # Pull down the trusted docker image local image=$i:$VERSION$IMAGE_TAG_SUFFIX - docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image + docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 # Get signature - curl -A "$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)" https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig + curl -A "$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)" https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig >> "$LOG_FILE" 2>&1 if [[ $? -ne 0 ]]; then - echo "Unable to pull signature file for $image" + echo "Unable to pull signature file for $image" >> "$LOG_FILE" 2>&1 exit 1 fi # Dump our hash values @@ -143,7 +148,7 @@ update_docker_containers() { echo "$DOCKERINSPECT" | jq ".[0].Created, .[0].RootFS.Layers" >> $SIGNPATH/$image.txt if [[ $? -ne 0 ]]; then - echo "Unable to inspect $image" + echo "Unable to inspect $image" >> "$LOG_FILE" 2>&1 exit 1 fi GPGTEST=$(gpg --verify $SIGNPATH/$image.sig $SIGNPATH/$image.txt 2>&1) @@ -153,15 +158,14 @@ update_docker_containers() { if [ -z "$HOSTNAME" ]; then HOSTNAME=$(hostname) fi - docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$image $HOSTNAME:5000/$IMAGEREPO/$image - docker push $HOSTNAME:5000/$IMAGEREPO/$image + docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$image $HOSTNAME:5000/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 + docker push $HOSTNAME:5000/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 fi else - echo "There is a problem downloading the $image image. Details: " - echo "" - echo $GPGTEST + echo "There is a problem downloading the $image image. Details: " >> "$LOG_FILE" 2>&1 + echo "" >> "$LOG_FILE" 2>&1 + echo $GPGTEST >> "$LOG_FILE" 2>&1 exit 1 fi done - } diff --git a/salt/domainstats/init.sls b/salt/domainstats/init.sls index 7716ddf83..965d87426 100644 --- a/salt/domainstats/init.sls +++ b/salt/domainstats/init.sls @@ -43,13 +43,13 @@ dstatslogdir: so-domainstatsimage: cmd.run: - - name: docker pull --disable-content-trust=false docker.io/{{ IMAGEREPO }}/so-domainstats:HH1.0.3 + - name: docker pull {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-domainstats:{{ VERSION }} so-domainstats: docker_container.running: - require: - so-domainstatsimage - - image: docker.io/{{ IMAGEREPO }}/so-domainstats:HH1.0.3 + - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-domainstats:{{ VERSION }} - hostname: domainstats - name: so-domainstats - user: domainstats diff --git a/salt/freqserver/init.sls b/salt/freqserver/init.sls index 5ff454bcc..f514353a1 100644 --- a/salt/freqserver/init.sls +++ b/salt/freqserver/init.sls @@ -43,13 +43,13 @@ freqlogdir: so-freqimage: cmd.run: - - name: docker pull --disable-content-trust=false docker.io/{{ IMAGEREPO }}/so-freqserver:HH1.0.3 + - name: docker pull {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-freqserver:{{ VERSION }} so-freq: docker_container.running: - require: - so-freqimage - - image: docker.io/{{ IMAGEREPO }}/so-freqserver:HH1.0.3 + - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-freqserver:{{ VERSION }} - hostname: freqserver - name: so-freqserver - user: freqserver diff --git a/salt/nodered/init.sls b/salt/nodered/init.sls index a594c23d9..c4fb8cb37 100644 --- a/salt/nodered/init.sls +++ b/salt/nodered/init.sls @@ -67,7 +67,7 @@ noderedlog: so-nodered: docker_container.running: - - image: {{ IMAGEREPO }}/so-nodered:HH1.2.2 + - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-nodered:{{ VERSION }} - interactive: True - binds: - /opt/so/conf/nodered/:/data:rw diff --git a/salt/registry/init.sls b/salt/registry/init.sls index c456aa0c4..43b9d8fa6 100644 --- a/salt/registry/init.sls +++ b/salt/registry/init.sls @@ -45,7 +45,7 @@ dockerregistryconf: # Install the registry container so-dockerregistry: docker_container.running: - - image: registry:latest + - image: ghcr.io/security-onion-solutions/registry:latest - hostname: so-registry - restart_policy: always - port_bindings: diff --git a/setup/so-functions b/setup/so-functions index 8ec78787e..273472f25 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -901,10 +901,7 @@ docker_seed_registry() { docker_seed_update_percent=25 - # Save output descriptors for use in docker_seed_registry_update function - exec 10>&1 20>&2 - - update_docker_containers 'netinstall' '' 'docker_seed_update' >> "$setup_log" 2>&1 + update_docker_containers 'netinstall' '' 'docker_seed_update' "$setup_log" else tar xvf /nsm/docker-registry/docker/registry.tar -C /nsm/docker-registry/docker >> "$setup_log" 2>&1 rm /nsm/docker-registry/docker/registry.tar >> "$setup_log" 2>&1 diff --git a/setup/so-setup b/setup/so-setup index 21c78cd92..381ef9bca 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -599,9 +599,9 @@ fi else set_progress_str 26 'Downloading containers from the internet' fi - import_registry_docker >> $setup_log 2>&1 + import_registry_docker >> $setup_log 2>&1 salt-call state.apply -l info registry >> $setup_log 2>&1 - docker_seed_registry 2>> "$setup_log" # ~ 60% when finished + docker_seed_registry # ~ 60% when finished set_progress_str 60 "$(print_salt_state_apply 'manager')" if [[ "$STRELKARULES" == 1 ]]; then From 3bae243915a681c7ca623245ec4460d99fcee18b Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 16 Nov 2020 15:20:00 -0500 Subject: [PATCH 4/8] Continued refactoring of bash --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 273472f25..2b103e396 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -893,7 +893,7 @@ docker_seed_registry() { if ! [ -f /nsm/docker-registry/docker/registry.tar ]; then if [ "$install_type" == 'IMPORT' ]; then container_list 'so-import' - elif [ "$install_type" != 'HELIXSENSOR' ]; then + elif [ "$install_type" == 'HELIXSENSOR' ]; then container_list 'so-helix' else container_list From 5ae78d4108cc339dc2892024400505092211fdae Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 16 Nov 2020 15:31:40 -0500 Subject: [PATCH 5/8] Install curl in order to test for cloud --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 2b103e396..cb88f8dcf 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -743,7 +743,7 @@ detect_os() { systemctl start NetworkManager; } >> "$setup_log" 2<&1 fi - apt-get install -y bc >> "$setup_log" 2>&1 + apt-get install -y bc curl >> "$setup_log" 2>&1 else echo "We were unable to determine if you are using a supported OS." From 1ec4af1a4d2c256fd94c2310dcb1eec4f182cf48 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 16 Nov 2020 15:41:15 -0500 Subject: [PATCH 6/8] Destroy the old registry before updating SO images --- salt/common/tools/sbin/soup | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 72e0e58df..1f3153d41 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -93,7 +93,12 @@ airgap_update_dockers() { docker load -i $AGDOCKER/registry_image.tar fi fi +} +update_registry() { + docker stop so-dockerregistry + docker rm so-dockerregistry + salt-call state.apply registry } check_airgap() { @@ -431,6 +436,7 @@ echo "Updating dockers to $NEWVERSION." if [ $is_airgap -eq 0 ]; then airgap_update_dockers else + update_registry update_docker_containers "soup" fi echo "" From 3cf8afc1ddf48eeb14155bdb03ee0c1dae7e067e Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 16 Nov 2020 16:39:54 -0500 Subject: [PATCH 7/8] Remove unused redirect descriptors and ensure gpg import output is not leaked to console --- salt/common/tools/sbin/so-image-common | 2 +- setup/so-functions | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index aefeade91..9fa1278ef 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -117,7 +117,7 @@ update_docker_containers() { fi # Let's make sure we have the public key - curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS | gpg --import - + curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS >> "$LOG_FILE" 2>&1 | gpg --import - >> "$LOG_FILE" 2>&1 rm -rf $SIGNPATH >> "$LOG_FILE" 2>&1 mkdir -p $SIGNPATH >> "$LOG_FILE" 2>&1 diff --git a/setup/so-functions b/setup/so-functions index cb88f8dcf..fb8b17a88 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -878,13 +878,7 @@ docker_seed_update() { fi ((docker_seed_update_percent=docker_seed_update_percent+percent_delta)) - # Backup current output descriptors and reset to saved descriptors from docker_seed_registry function - exec 8>&1 9>&2 1>&10 2>&20 - set_progress_str "$docker_seed_update_percent" "Downloading $name" - - # Restore current output descriptors and remove backups - exec 1>&8- 2>&9- } docker_seed_registry() { From 4311f661102daf403077d87be7b576eb95781a48 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 16 Nov 2020 16:58:09 -0500 Subject: [PATCH 8/8] Remove unnecessary redirect --- salt/common/tools/sbin/so-image-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 9fa1278ef..165c20528 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -117,7 +117,7 @@ update_docker_containers() { fi # Let's make sure we have the public key - curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS >> "$LOG_FILE" 2>&1 | gpg --import - >> "$LOG_FILE" 2>&1 + curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS | gpg --import - >> "$LOG_FILE" 2>&1 rm -rf $SIGNPATH >> "$LOG_FILE" 2>&1 mkdir -p $SIGNPATH >> "$LOG_FILE" 2>&1