diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index a4eeb5239..165c20528 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -19,29 +19,30 @@ IMAGEREPO=securityonion container_list() { - MANAGERCHECK=so-unknown - if [ -f /etc/salt/grains ]; then - MANAGERCHECK=$(cat /etc/salt/grains | grep role | awk '{print $2}') + MANAGERCHECK=$1 + if [ -z "$MANAGERCHECK" ]; then + MANAGERCHECK=so-unknown + if [ -f /etc/salt/grains ]; then + MANAGERCHECK=$(cat /etc/salt/grains | grep role | awk '{print $2}') + fi fi if [ $MANAGERCHECK == 'so-import' ]; then - TRUSTED_CONTAINERS=( \ - "so-idstools" \ - "so-nginx" \ - "so-filebeat" \ - "so-suricata" \ - "so-soc" \ + TRUSTED_CONTAINERS=( \ "so-elasticsearch" \ + "so-filebeat" \ + "so-idstools" \ "so-kibana" \ "so-kratos" \ - "so-suricata" \ - "so-registry" \ + "so-nginx" \ "so-pcaptools" \ + "so-soc" \ + "so-steno" \ + "so-suricata" \ "so-zeek" ) elif [ $MANAGERCHECK != 'so-helix' ]; then - TRUSTED_CONTAINERS=( \ + TRUSTED_CONTAINERS=( \ "so-acng" \ - "so-thehive-cortex" \ "so-curator" \ "so-domainstats" \ "so-elastalert" \ @@ -65,18 +66,19 @@ container_list() { "so-soc" \ "so-soctopus" \ "so-steno" \ - "so-strelka-frontend" \ - "so-strelka-manager" \ "so-strelka-backend" \ "so-strelka-filestream" \ + "so-strelka-frontend" \ + "so-strelka-manager" \ "so-suricata" \ "so-telegraf" \ "so-thehive" \ + "so-thehive-cortex" \ "so-thehive-es" \ "so-wazuh" \ "so-zeek" ) else - TRUSTED_CONTAINERS=( \ + TRUSTED_CONTAINERS=( \ "so-filebeat" \ "so-idstools" \ "so-logstash" \ @@ -90,16 +92,22 @@ container_list() { } update_docker_containers() { - CURLTYPE=$1 - IMAGE_TAG_SUFFIX=$2 + local CURLTYPE=$1 + local IMAGE_TAG_SUFFIX=$2 + local PROGRESS_CALLBACK=$3 + local LOG_FILE=$4 - CONTAINER_REGISTRY=quay.io - SIGNPATH=/root/sosigs + local CONTAINER_REGISTRY=quay.io + local SIGNPATH=/root/sosigs if [ -z "$CURLTYPE" ]; then CURLTYPE=unknown fi + if [ -z "$LOG_FILE" ]; then + LOG_FILE=/dev/tty + fi + # Recheck the version for scenarios were the VERSION wasn't known before this script was imported set_version set_os @@ -109,50 +117,55 @@ update_docker_containers() { fi # Let's make sure we have the public key - curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS | gpg --import - + curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS | gpg --import - >> "$LOG_FILE" 2>&1 - rm -rf $SIGNPATH - mkdir -p $SIGNPATH + rm -rf $SIGNPATH >> "$LOG_FILE" 2>&1 + mkdir -p $SIGNPATH >> "$LOG_FILE" 2>&1 # Download the containers from the interwebs for i in "${TRUSTED_CONTAINERS[@]}" do + if [ -z "$PROGRESS_CALLBACK" ]; then + echo "Downloading $i" >> "$LOG_FILE" 2>&1 + else + $PROGRESS_CALLBACK $i + fi + # Pull down the trusted docker image - echo "Downloading $i" - docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION$IMAGE_TAG_SUFFIX + local image=$i:$VERSION$IMAGE_TAG_SUFFIX + docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 # Get signature - curl -A "$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)" https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$i:$VERSION$IMAGE_TAG_SUFFIX.sig + curl -A "$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)" https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig >> "$LOG_FILE" 2>&1 if [[ $? -ne 0 ]]; then - echo "Unable to pull signature file for $i:$VERSION$IMAGE_TAG_SUFFIX" + echo "Unable to pull signature file for $image" >> "$LOG_FILE" 2>&1 exit 1 fi # Dump our hash values - DOCKERINSPECT=$(docker inspect $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION$IMAGE_TAG_SUFFIX) + DOCKERINSPECT=$(docker inspect $CONTAINER_REGISTRY/$IMAGEREPO/$image) - echo "$DOCKERINSPECT" | jq ".[0].RepoDigests[] | select(. | contains(\"$CONTAINER_REGISTRY\"))" > $SIGNPATH/$i:$VERSION$IMAGE_TAG_SUFFIX.txt - echo "$DOCKERINSPECT" | jq ".[0].Created, .[0].RootFS.Layers" >> $SIGNPATH/$i:$VERSION$IMAGE_TAG_SUFFIX.txt + echo "$DOCKERINSPECT" | jq ".[0].RepoDigests[] | select(. | contains(\"$CONTAINER_REGISTRY\"))" > $SIGNPATH/$image.txt + echo "$DOCKERINSPECT" | jq ".[0].Created, .[0].RootFS.Layers" >> $SIGNPATH/$image.txt if [[ $? -ne 0 ]]; then - echo "Unable to inspect $i:$VERSION$IMAGE_TAG_SUFFIX" + echo "Unable to inspect $image" >> "$LOG_FILE" 2>&1 exit 1 fi - GPGTEST=$(gpg --verify $SIGNPATH/$i:$VERSION$IMAGE_TAG_SUFFIX.sig $SIGNPATH/$i:$VERSION$IMAGE_TAG_SUFFIX.txt 2>&1) + GPGTEST=$(gpg --verify $SIGNPATH/$image.sig $SIGNPATH/$image.txt 2>&1) if [[ $? -eq 0 ]]; then if [[ -z "$SKIP_TAGPUSH" ]]; then # Tag it with the new registry destination if [ -z "$HOSTNAME" ]; then HOSTNAME=$(hostname) fi - docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION$IMAGE_TAG_SUFFIX $HOSTNAME:5000/$IMAGEREPO/$i:$VERSION$IMAGE_TAG_SUFFIX - docker push $HOSTNAME:5000/$IMAGEREPO/$i:$VERSION$IMAGE_TAG_SUFFIX + docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$image $HOSTNAME:5000/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 + docker push $HOSTNAME:5000/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 fi else - echo "There is a problem downloading the $i:$VERSION$IMAGE_TAG_SUFFIX image. Details: " - echo "" - echo $GPGTEST + echo "There is a problem downloading the $image image. Details: " >> "$LOG_FILE" 2>&1 + echo "" >> "$LOG_FILE" 2>&1 + echo $GPGTEST >> "$LOG_FILE" 2>&1 exit 1 fi done - } diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 72e0e58df..1f3153d41 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -93,7 +93,12 @@ airgap_update_dockers() { docker load -i $AGDOCKER/registry_image.tar fi fi +} +update_registry() { + docker stop so-dockerregistry + docker rm so-dockerregistry + salt-call state.apply registry } check_airgap() { @@ -431,6 +436,7 @@ echo "Updating dockers to $NEWVERSION." if [ $is_airgap -eq 0 ]; then airgap_update_dockers else + update_registry update_docker_containers "soup" fi echo "" diff --git a/salt/domainstats/init.sls b/salt/domainstats/init.sls index 7716ddf83..965d87426 100644 --- a/salt/domainstats/init.sls +++ b/salt/domainstats/init.sls @@ -43,13 +43,13 @@ dstatslogdir: so-domainstatsimage: cmd.run: - - name: docker pull --disable-content-trust=false docker.io/{{ IMAGEREPO }}/so-domainstats:HH1.0.3 + - name: docker pull {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-domainstats:{{ VERSION }} so-domainstats: docker_container.running: - require: - so-domainstatsimage - - image: docker.io/{{ IMAGEREPO }}/so-domainstats:HH1.0.3 + - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-domainstats:{{ VERSION }} - hostname: domainstats - name: so-domainstats - user: domainstats diff --git a/salt/freqserver/init.sls b/salt/freqserver/init.sls index 5ff454bcc..f514353a1 100644 --- a/salt/freqserver/init.sls +++ b/salt/freqserver/init.sls @@ -43,13 +43,13 @@ freqlogdir: so-freqimage: cmd.run: - - name: docker pull --disable-content-trust=false docker.io/{{ IMAGEREPO }}/so-freqserver:HH1.0.3 + - name: docker pull {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-freqserver:{{ VERSION }} so-freq: docker_container.running: - require: - so-freqimage - - image: docker.io/{{ IMAGEREPO }}/so-freqserver:HH1.0.3 + - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-freqserver:{{ VERSION }} - hostname: freqserver - name: so-freqserver - user: freqserver diff --git a/salt/nodered/init.sls b/salt/nodered/init.sls index a594c23d9..c4fb8cb37 100644 --- a/salt/nodered/init.sls +++ b/salt/nodered/init.sls @@ -67,7 +67,7 @@ noderedlog: so-nodered: docker_container.running: - - image: {{ IMAGEREPO }}/so-nodered:HH1.2.2 + - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-nodered:{{ VERSION }} - interactive: True - binds: - /opt/so/conf/nodered/:/data:rw diff --git a/salt/registry/init.sls b/salt/registry/init.sls index c456aa0c4..43b9d8fa6 100644 --- a/salt/registry/init.sls +++ b/salt/registry/init.sls @@ -45,7 +45,7 @@ dockerregistryconf: # Install the registry container so-dockerregistry: docker_container.running: - - image: registry:latest + - image: ghcr.io/security-onion-solutions/registry:latest - hostname: so-registry - restart_policy: always - port_bindings: diff --git a/setup/so-functions b/setup/so-functions index 642ae5004..fb8b17a88 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -743,7 +743,7 @@ detect_os() { systemctl start NetworkManager; } >> "$setup_log" 2<&1 fi - apt-get install -y bc >> "$setup_log" 2>&1 + apt-get install -y bc curl >> "$setup_log" 2>&1 else echo "We were unable to determine if you are using a supported OS." @@ -870,116 +870,32 @@ docker_registry() { } +docker_seed_update() { + local name=$1 + local percent_delta=1 + if [ "$install_type" == 'HELIXSENSOR' ]; then + percent_delta=6 + fi + ((docker_seed_update_percent=docker_seed_update_percent+percent_delta)) + + set_progress_str "$docker_seed_update_percent" "Downloading $name" +} + docker_seed_registry() { local VERSION="$SOVERSION" if ! [ -f /nsm/docker-registry/docker/registry.tar ]; then - if [ "$install_type" == 'IMPORT' ]; then - local TRUSTED_CONTAINERS=(\ - "so-idstools" \ - "so-nginx" \ - "so-filebeat" \ - "so-suricata" \ - "so-soc" \ - "so-steno" \ - "so-elasticsearch" \ - "so-kibana" \ - "so-kratos" \ - "so-suricata" \ - "so-pcaptools" \ - "so-zeek" - ) + if [ "$install_type" == 'IMPORT' ]; then + container_list 'so-import' + elif [ "$install_type" == 'HELIXSENSOR' ]; then + container_list 'so-helix' else - local TRUSTED_CONTAINERS=(\ - "so-nginx" \ - "so-filebeat" \ - "so-logstash" \ - "so-idstools" \ - "so-redis" \ - "so-steno" \ - "so-suricata" \ - "so-telegraf" \ - "so-zeek" - ) + container_list fi - if [ "$install_type" != 'HELIXSENSOR' ] && [ "$install_type" != 'IMPORT' ]; then - TRUSTED_CONTAINERS=("${TRUSTED_CONTAINERS[@]}" \ - "so-acng" \ - "so-thehive-cortex" \ - "so-curator" \ - "so-domainstats" \ - "so-elastalert" \ - "so-elasticsearch" \ - "so-fleet" \ - "so-fleet-launcher" \ - "so-freqserver" \ - "so-grafana" \ - "so-influxdb" \ - "so-kibana" \ - "so-minio" \ - "so-mysql" \ - "so-pcaptools" \ - "so-playbook" \ - "so-soc" \ - "so-kratos" \ - "so-soctopus" \ - "so-steno" \ - "so-strelka-frontend" \ - "so-strelka-manager" \ - "so-strelka-backend" \ - "so-strelka-filestream" \ - "so-thehive" \ - "so-thehive-es" \ - "so-wazuh" - ) - fi - local percent=25 - # Let's make sure we have the public key - curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS | gpg --import - - - SIGNPATH=/root/sosigs - rm -rf $SIGNPATH - mkdir -p $SIGNPATH - if [ -z "$BRANCH" ]; then - BRANCH="master" - fi - for i in "${TRUSTED_CONTAINERS[@]}"; do - if [ "$install_type" != 'HELIXSENSOR' ]; then ((percent=percent+1)); else ((percent=percent+6)); fi - # Pull down the trusted docker image - set_progress_str "$percent" "Downloading $i:$VERSION" - { - echo "Downloading $i" - docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION - - # Get signature - curl -A "netinstall/$OS/$(uname -r)" https://sigs.securityonion.net/$VERSION/$i:$VERSION.sig --output $SIGNPATH/$i:$VERSION.sig - if [[ $? -ne 0 ]]; then - echo "Unable to pull signature file for $i:$VERSION" - exit 1 - fi - # Dump our hash values - DOCKERINSPECT=$(docker inspect $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION) - - echo "$DOCKERINSPECT" | jq ".[0].RepoDigests[] | select(. | contains(\"$CONTAINER_REGISTRY\"))" > $SIGNPATH/$i:$VERSION.txt - echo "$DOCKERINSPECT" | jq ".[0].Created, .[0].RootFS.Layers" >> $SIGNPATH/$i:$VERSION.txt - - if [[ $? -ne 0 ]]; then - echo "Unable to inspect $i" - exit 1 - fi - GPGTEST=$(gpg --verify $SIGNPATH/$i:$VERSION.sig $SIGNPATH/$i:$VERSION.txt 2>&1) - if [[ $? -eq 0 ]]; then - # Tag it with the new registry destination - docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$i:$VERSION $HOSTNAME:5000/$IMAGEREPO/$i:$VERSION - docker push $HOSTNAME:5000/$IMAGEREPO/$i:$VERSION - else - echo "There is a problem downloading the $i image. Details: " - echo "" - echo $GPGTEST - exit 1 - fi - } >> "$setup_log" 2>&1 - done + + docker_seed_update_percent=25 + + update_docker_containers 'netinstall' '' 'docker_seed_update' "$setup_log" else tar xvf /nsm/docker-registry/docker/registry.tar -C /nsm/docker-registry/docker >> "$setup_log" 2>&1 rm /nsm/docker-registry/docker/registry.tar >> "$setup_log" 2>&1 @@ -1006,10 +922,10 @@ firewall_generate_templates() { local firewall_pillar_path=$local_salt_dir/salt/firewall mkdir -p "$firewall_pillar_path" - cp ../files/firewall/* /opt/so/saltstack/local/salt/firewall/ >> "$setup_log" 2>&1 + cp ../files/firewall/* /opt/so/saltstack/local/salt/firewall/ >> "$setup_log" 2>&1 - for i in analyst beats_endpoint sensor manager minion osquery_endpoint search_node wazuh_endpoint; do - $default_salt_dir/salt/common/tools/sbin/so-firewall includehost "$i" 127.0.0.1 + for i in analyst beats_endpoint sensor manager minion osquery_endpoint search_node wazuh_endpoint; do + $default_salt_dir/salt/common/tools/sbin/so-firewall includehost "$i" 127.0.0.1 done } diff --git a/setup/so-setup b/setup/so-setup index 83bb8ceed..065ba548f 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -599,9 +599,9 @@ fi else set_progress_str 26 'Downloading containers from the internet' fi - import_registry_docker >> $setup_log 2>&1 + import_registry_docker >> $setup_log 2>&1 salt-call state.apply -l info registry >> $setup_log 2>&1 - docker_seed_registry 2>> "$setup_log" # ~ 60% when finished + docker_seed_registry # ~ 60% when finished set_progress_str 60 "$(print_salt_state_apply 'manager')" if [[ "$STRELKARULES" == 1 ]]; then