ignore repo error that occurs on new iso installs for remote minions

ES 9.0.8

VERSION

@@ -1 +1 @@
-2.4.0-foxtrot
+2.4.210

salt/manager/tools/sbin/soup

@@ -1681,9 +1681,11 @@ verify_es_version_compatibility() {
         create_intermediate_upgrade_verification_script $es_verification_script
     fi
 
-    # create script using version in statefile
     local es_required_version_statefile_value=$(cat $es_required_version_statefile)
-    timeout --foreground 3600 bash "$es_verification_script" "$es_required_version_statefile_value" "$es_required_version_statefile"
+    echo -e "\n##############################################################################################################################\n"
+    echo "A previously required intermediate Elasticsearch upgrade was detected. Verifying that all Searchnodes/Heavynodes have successfully upgraded Elasticsearch to $es_required_version_statefile_value before proceeding with soup to avoid potential data loss!"
+    # create script using version in statefile
+    timeout --foreground 4000 bash "$es_verification_script" "$es_required_version_statefile_value" "$es_required_version_statefile"
     if [[ $? -ne 0 ]]; then
         echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
 
@@ -1692,7 +1694,7 @@ verify_es_version_compatibility() {
         echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
         exit 161
     fi
-
+    echo -e "\n##############################################################################################################################\n"
   fi
 
   if [[ " ${es_upgrade_map[$es_version]} " =~ " $target_es_version " || "$es_version" == "$target_es_version" ]]; then
@@ -1728,7 +1730,7 @@ verify_es_version_compatibility() {
       exec bash -c "BRANCH=$next_step_so_version soup -y && BRANCH=$next_step_so_version soup -y && \
        echo -e \"\n##############################################################################################################################\n\" && \
        echo -e \"Verifying Elasticsearch was successfully upgraded to ${compatible_versions##* } across the grid. This part can take a while as Searchnodes/Heavynodes sync up with the Manager! \n\nOnce verification completes the next soup will begin automatically. If verification takes longer than 1 hour it will stop waiting and your grid will remain at $next_step_so_version. Allowing for all Searchnodes/Heavynodes to upgrade Elasticsearch to the required version on their own time.\n\" \
-       && timeout --foreground 3600 bash /tmp/so_intermediate_upgrade_verification.sh ${compatible_versions##* } $es_required_version_statefile && \
+       && timeout --foreground 4000 bash /tmp/so_intermediate_upgrade_verification.sh ${compatible_versions##* } $es_required_version_statefile && \
        echo -e \"\n##############################################################################################################################\n\" \
        && BRANCH=$originally_requested_so_version soup -y && BRANCH=$originally_requested_so_version soup -y"
     fi
@@ -1770,10 +1772,10 @@ create_intermediate_upgrade_verification_script() {
         local retries=20
         local retry_count=0
         local delay=180
-        local success=1
+
         while [[ $retry_count -lt $retries ]]; do
             # keep stderr with variable for logging
-            heavynode_versions=$(salt -C 'G@role:so-heavynode' cmd.run 'so-elasticsearch-query / --retry 3 --retry-delay 10 | jq ".version.number"' shell=/bin/bash --out=json 2>&1)
+            heavynode_versions=$(salt -C 'G@role:so-heavynode' cmd.run 'so-elasticsearch-query / --retry 3 --retry-delay 10 | jq ".version.number"' shell=/bin/bash --out=json 2> /dev/null)
             local exit_status=$?
 
             # Check that all heavynodes returned good data
@@ -1789,7 +1791,7 @@ create_intermediate_upgrade_verification_script() {
 
                     return 0
                 else
-                    echo "One or more heavynodes is not at the expected Elasticsearch version $EXPECTED_ES_VERSION. Rechecking in $delay seconds. Attempt $((retry_count + 1)) of $retries."
+                    echo "One or more heavynodes are not at the expected Elasticsearch version $EXPECTED_ES_VERSION. Rechecking in $delay seconds. Attempt $((retry_count + 1)) of $retries."
                     ((retry_count++))
                     sleep $delay
 
@@ -1814,11 +1816,10 @@ create_intermediate_upgrade_verification_script() {
         local retries=20
         local retry_count=0
         local delay=180
-        local success=1
 
         while [[ $retry_count -lt $retries ]]; do
             # keep stderr with variable for logging
-            cluster_versions=$(so-elasticsearch-query _nodes/_all/version --retry 5 --retry-delay 10 2>&1)
+            cluster_versions=$(so-elasticsearch-query _nodes/_all/version --retry 5 --retry-delay 10 --fail 2>&1)
             local exit_status=$?
 
             if [[ $exit_status -ne 0 ]]; then

salt/sensoroni/files/templates/reports/standard/case_report.md

@@ -130,4 +130,42 @@ Security Onion Case Report
 | ---- | ---- | ------ | --------- |
 {{ range sortHistory "CreateTime" "asc" .History -}}
 | {{formatDateTime "Mon Jan 02 15:04:05 -0700 2006" .CreateTime}} | {{getUserDetail "email" .UserId}} | {{.Kind}} | {{.Operation}} |
+{{end}}
+
+## Attached Onion AI Sessions
+
+{{ range $idx, $session := sortAssistantSessionDetails "CreateTime" "desc" .AssistantSessions }}
+
+#### Session {{ add $idx 1 }}
+
+**Session ID:** {{$session.Session.SessionId}}
+
+**Title:** {{$session.Session.Title}}
+
+**User ID:** {{getUserDetail "email" $session.Session.UserId}}
+
+**Created:** {{formatDateTime "Mon Jan 02 15:04:05 -0700 2006" $session.Session.CreateTime}}
+
+**Updated:** {{formatDateTime "Mon Jan 02 15:04:05 -0700 2006" $session.Session.UpdateTime}}
+
+{{ if $session.Session.DeleteTime }}
+**Deleted:** {{ formatDateTime "Mon Jan 02 15:04:05 -0700 2006" $session.Session.DeleteTime}}
+{{ end }}
+
+#### Messages
+
+{{ range $index, $msg := sortAssistantMessages "CreateTime" "asc" $session.History }}
+{{ range $i, $block := $msg.Message.ContentBlocks }}
+
+{{ if eq $block.Type "text" }}
+
+**Role:** {{$msg.Message.Role}}
+
+{{ stripEmoji $block.Text }}
+
+---
+
+{{ end }}{{ end }}
+
+{{end}}
 {{end}}

setup/so-verify

@@ -69,6 +69,7 @@ log_has_errors() {
         grep -vE "Running scope as unit" | \
         grep -vE "securityonion-resources/sigma/stable" | \
         grep -vE "remove_failed_vm.sls" | \
+        grep -vE "Error: Failed to download metadata for repo 'securityonion': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried" | \
         grep -vE "log-.*-pipeline_failed_attempts" &> "$error_log"
     
     if [[ $? -eq 0 ]]; then