Merge pull request #15539 from Security-Onion-Solutions/jertel/wip

prepare for nextgen docs

.github/DISCUSSION_TEMPLATE/2-4.yml

@@ -96,7 +96,7 @@ body:
     attributes:
       label: Hardware Specs
       description: >
-        Does your hardware meet or exceed the minimum requirements for your installation type as shown at https://docs.securityonion.net/en/2.4/hardware.html?
+        Does your hardware meet or exceed the minimum requirements for your installation type as shown at https://securityonion.net/docs/hardware?
       options:
         -
         - Meets minimum requirements

DOWNLOAD_AND_VERIFY_ISO.md

@@ -50,4 +50,4 @@ Primary key fingerprint: C804 A93D 36BE 0C73 3EA1  9644 7C10 60B7 FE50 7013
 If it fails to verify, try downloading again. If it still fails to verify, try downloading from another computer or another network.
 
 Once you've verified the ISO image, you're ready to proceed to our Installation guide:  
-https://docs.securityonion.net/en/2.4/installation.html
+https://securityonion.net/docs/installation

README.md

@@ -27,24 +27,24 @@ Config
 
 ### Release Notes
 
-https://docs.securityonion.net/en/2.4/release-notes.html
+https://securityonion.net/docs/release-notes
 
 ### Requirements
 
-https://docs.securityonion.net/en/2.4/hardware.html
+https://securityonion.net/docs/hardware
 
 ### Download
 
-https://docs.securityonion.net/en/2.4/download.html
+https://securityonion.net/docs/download
 
 ### Installation
 
-https://docs.securityonion.net/en/2.4/installation.html
+https://securityonion.net/docs/installation
 
 ### FAQ
 
-https://docs.securityonion.net/en/2.4/faq.html
+https://securityonion.net/docs/faq
 
 ### Feedback
 
-https://docs.securityonion.net/en/2.4/community-support.html
+https://securityonion.net/docs/community-support

salt/common/tools/sbin/so-checkin

@@ -10,7 +10,7 @@
 cat << EOF
 
 so-checkin will run a full salt highstate to apply all salt states. If a highstate is already running, this request will be queued and so it may pause for a few minutes before you see any more output. For more information about so-checkin and salt, please see:
-https://docs.securityonion.net/en/2.4/salt.html
+https://securityonion.net/docs/salt
 
 EOF
 

salt/common/tools/sbin/so-common

@@ -10,7 +10,7 @@
 # and since this same logic is required during installation, it's included in this file.
 
 DEFAULT_SALT_DIR=/opt/so/saltstack/default
-DOC_BASE_URL="https://docs.securityonion.net/en/2.4"
+DOC_BASE_URL="https://securityonion.net/docs"
 
 if [ -z $NOROOT ]; then
 	# Check for prerequisites

salt/common/tools/sbin_jinja/so-desktop-install

@@ -6,7 +6,7 @@
 # Elastic License 2.0.
 
 source /usr/sbin/so-common
-doc_desktop_url="$DOC_BASE_URL/desktop.html"
+doc_desktop_url="$DOC_BASE_URL/desktop"
 
 {# we only want the script to install the desktop if it is OEL -#}
 {% if grains.os == 'OEL' -%}

salt/manager/tools/sbin/soup

@@ -52,7 +52,7 @@ check_err() {
       ;;
       28)
         echo 'No space left on device'
-        echo "Likely ran out of space on disk, please review hardware requirements for Security Onion: $DOC_BASE_URL/hardware.html"
+        echo "Likely ran out of space on disk, please review hardware requirements for Security Onion: $DOC_BASE_URL/hardware"
       ;;
       30)
         echo 'Read-only file system'
@@ -1053,7 +1053,7 @@ used and enables informed prioritization of future development.
 
 Adjust this setting at anytime via the SOC Configuration screen.
 
-Documentation: https://docs.securityonion.net/en/2.4/telemetry.html
+Documentation: https://securityonion.net/docs/telemetry
 
 ASSIST_EOF
 
@@ -1201,7 +1201,7 @@ suricata_idstools_removal_pre() {
 install -d -o 939 -g 939 -m 755 /opt/so/conf/soc/fingerprints
 install -o 939 -g 939 -m 644 /dev/null /opt/so/conf/soc/fingerprints/suricataengine.syncBlock
 cat > /opt/so/conf/soc/fingerprints/suricataengine.syncBlock << EOF
-Suricata ruleset sync is blocked until this file is removed. **CRITICAL** Make sure that you have manually added any custom Suricata rulesets via SOC config before removing this file - review the documentation for more details: https://docs.securityonion.net/en/2.4/nids.html#sync-block
+Suricata ruleset sync is blocked until this file is removed. **CRITICAL** Make sure that you have manually added any custom Suricata rulesets via SOC config before removing this file - review the documentation for more details: https://securityonion.net/docs/nids
 EOF
 
 # Remove possible symlink & create salt local rules dir
@@ -1825,6 +1825,25 @@ verify_es_version_compatibility() {
 
 }
 
+wait_for_salt_minion_with_restart() {
+    local minion="$1"
+    local max_wait="${2:-60}"
+    local interval="${3:-3}"
+    local logfile="$4"
+    
+    wait_for_salt_minion "$minion" "$max_wait" "$interval" "$logfile"
+    local result=$?
+    
+    if [[ $result -ne 0 ]]; then
+        echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - salt-minion not ready, attempting restart..."
+        systemctl_func "restart" "salt-minion"
+        wait_for_salt_minion "$minion" "$max_wait" "$interval" "$logfile"
+        result=$?
+    fi
+    
+    return $result
+}
+
 run_airgap_intermediate_upgrade() {
     local originally_requested_so_version=$(cat $UPDATE_DIR/VERSION)
     # preserve ISOLOC value, so we can try to use it post intermediate upgrade
@@ -2075,7 +2094,7 @@ apply_hotfix() {
       mv /etc/pki/managerssl.crt /etc/pki/managerssl.crt.old
       mv /etc/pki/managerssl.key /etc/pki/managerssl.key.old
       systemctl_func "start" "salt-minion"
-      wait_for_salt_minion "$MINIONID" "120" "4" "$SOUP_LOG" || fail "Salt minion was not running or ready."
+       wait_for_salt_minion_with_restart "$MINIONID" "60" "3" "$SOUP_LOG" || fail "Salt minion was not running or ready."
     fi
   else
    echo "No actions required. ($INSTALLEDVERSION/$HOTFIXVERSION)"
@@ -2275,7 +2294,7 @@ main() {
     echo ""
     echo "Running a highstate. This could take several minutes."
     set +e
-    wait_for_salt_minion "$MINIONID" "120" "4" "$SOUP_LOG" || fail "Salt minion was not running or ready."
+    wait_for_salt_minion_with_restart "$MINIONID" "60" "3" "$SOUP_LOG" || fail "Salt minion was not running or ready."
     highstate
     set -e
 
@@ -2288,7 +2307,7 @@ main() {
     check_saltmaster_status
 
     echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes."
-    wait_for_salt_minion "$MINIONID" "120" "4" "$SOUP_LOG" || fail "Salt minion was not running or ready."
+    wait_for_salt_minion_with_restart "$MINIONID" "60" "3" "$SOUP_LOG" || fail "Salt minion was not running or ready."
 
     # Stop long-running scripts to allow potentially updated scripts to load on the next execution.
     if pgrep salt-relay.sh > /dev/null 2>&1; then
@@ -2359,7 +2378,7 @@ Each minion is on a random 15 minute check-in period and things like network ban
 
 If it looks like you’re missing data after the upgrade, please avoid restarting services and instead make sure at least one search node has completed its upgrade. The best way to do this is to run 'sudo salt-call state.highstate' from a search node and make sure there are no errors. Typically if it works on one node it will work on the rest. Sensor nodes are less complex and will update as they check in so you can monitor those from the Grid section of SOC.
 
-For more information, please see $DOC_BASE_URL/soup.html#distributed-deployments.
+For more information, please see $DOC_BASE_URL/soup#distributed-deployments.
 
 EOF
 
@@ -2373,7 +2392,7 @@ EOF
 
 A custom Elasticsearch configuration has been found at /opt/so/saltstack/local/elasticsearch/files/elasticsearch.yml. This file is no longer referenced in Security Onion versions >= 2.3.80.
 
-If you still need those customizations, you'll need to manually migrate them to the new Elasticsearch config as shown at $DOC_BASE_URL/elasticsearch.html.
+If you still need those customizations, you'll need to manually migrate them to the new Elasticsearch config as shown at $DOC_BASE_URL/elasticsearch.
 
 EOF
 
@@ -2434,7 +2453,7 @@ if [[ -z $UNATTENDED ]]; then
 SOUP - Security Onion UPdater
 
 Please review the following for more information about the update process and recent updates:
-$DOC_BASE_URL/soup.html
+$DOC_BASE_URL/soup
 https://blog.securityonion.net
 
 WARNING: If you run soup via an SSH session and that SSH session terminates, then any processes running in that session would terminate. You should avoid leaving soup unattended especially if the machine you are SSHing from is configured to sleep after a period of time. You might also consider using something like screen or tmux so that if your SSH session terminates, the processes will continue running on the server.

salt/nginx/etc/nginx.conf

@@ -181,7 +181,7 @@ http {
 		ssl_prefer_server_ciphers on;
 		ssl_protocols TLSv1.2 TLSv1.3;
 
-		location ~* (^/login/.*|^/js/.*|^/css/.*|^/images/.*) {
+		location ~* (^/login/.*|^/js/.*|^/css/.*|^/images/.*|^/pages/.*|^/docs/.*) {
 			proxy_pass            http://{{ GLOBALS.manager }}:9822;
 			proxy_read_timeout    90;
 			proxy_connect_timeout 90;
@@ -213,6 +213,9 @@ http {
 			proxy_buffering         off;
 			proxy_cache             off;
 			proxy_request_buffering off;
+			add_header              Cache-Control "no-cache, no-store, must-revalidate";
+			add_header              Pragma "no-cache";
+			add_header              Expires "0";
 		}
 
 		location ~ ^/auth/.*?(login|oidc/callback) {

salt/soc/defaults.yaml

@@ -1632,7 +1632,7 @@ soc:
                 sourceType: directory
             airgap:
               - name: Emerging-Threats
-                description: "Emerging Threats ruleset - To enable ET Pro on Airgap, review the documentation at https://docs.securityonion.net/suricata"
+                description: "Emerging Threats ruleset - To enable ET Pro on Airgap, review the documentation at https://securityonion.net/docs/suricata"
                 licenseKey: ""
                 enabled: true
                 sourceType: directory

setup/so-verify

@@ -69,6 +69,8 @@ log_has_errors() {
         grep -vE "Running scope as unit" | \
         grep -vE "securityonion-resources/sigma/stable" | \
         grep -vE "remove_failed_vm.sls" | \
+        grep -vE "failed to copy: httpReadSeeker" | \
+        grep -vE "Error response from daemon: failed to resolve reference" | \
         grep -vE "log-.*-pipeline_failed_attempts" &> "$error_log"
     
     if [[ $? -eq 0 ]]; then

setup/so-whiptail

@@ -14,7 +14,7 @@ whiptail_airgap() {
 	[[ $is_manager || $is_import ]] && node_str='manager'
 
 	INTERWEBS=$(whiptail --title "$whiptail_title" --menu \
-	"How should this $node_str be installed?\n\nFor more information, please see:\n$DOC_BASE_URL/airgap.html" 13 70 2 \
+	"How should this $node_str be installed?\n\nFor more information, please see:\n$DOC_BASE_URL/airgap" 13 70 2 \
 	"Standard    " "This $node_str has access to the Internet"  \
 	"Airgap      " "This $node_str does not have access to the Internet" 3>&1 1>&2 2>&3 )
 
@@ -156,7 +156,7 @@ whiptail_accept_telemetry() {
 
 	Adjust this setting at anytime via the SOC Configuration screen.
 
-	Documentation: https://docs.securityonion.net/en/2.4/telemetry.html
+	Documentation: https://securityonion.net/docs/telemetry
 
 	Enable SOC Telemetry to help improve future releases?
 	EOM
@@ -289,7 +289,7 @@ whiptail_storage_requirements() {
 
 		You need ${needed_val} to meet minimum requirements.
 
-		Visit $DOC_BASE_URL/hardware.html for more information.
+		Visit $DOC_BASE_URL/hardware for more information.
 
 		Select YES to continue anyway, or select NO to cancel.
 	EOM
@@ -588,7 +588,7 @@ whiptail_install_type() {
 	# What kind of install are we doing?
 	if [[ "$OSVER" != "focal" ]]; then
 		install_type=$(whiptail --title "$whiptail_title" --menu \
-		"What kind of installation would you like to do?\n\nFor more information, please see:\n$DOC_BASE_URL/architecture.html" 18 65 5 \
+		"What kind of installation would you like to do?\n\nFor more information, please see:\n$DOC_BASE_URL/architecture" 18 65 5 \
 			"IMPORT" "Import PCAP or log files " \
 			"EVAL" "Evaluation mode (not for production) " \
 			"STANDALONE" "Standalone production install " \
@@ -598,7 +598,7 @@ whiptail_install_type() {
 		)
 	elif [[ "$OSVER" == "focal" ]]; then
 		install_type=$(whiptail --title "$whiptail_title" --menu \
-		"What kind of installation would you like to do?\n\nFor more information, please see:\n$DOC_BASE_URL/architecture.html" 18 65 5 \
+		"What kind of installation would you like to do?\n\nFor more information, please see:\n$DOC_BASE_URL/architecture" 18 65 5 \
 			"DISTRIBUTED" "Distributed install submenu " \
 			3>&1 1>&2 2>&3
 		)
@@ -649,7 +649,7 @@ whiptail_install_type_dist_new() {
 	read -r -d '' mngr_msg <<- EOM
 	Choose a distributed manager type to start a new grid.
 	
-	See $DOC_BASE_URL/architecture.html for details.
+	See $DOC_BASE_URL/architecture for details.
 
 	Note: MANAGER is the recommended option for most users. MANAGERSEARCH should only be used in very specific situations.
 	EOM
@@ -670,7 +670,7 @@ whiptail_install_type_dist_existing() {
 
 	local node_msg
 	read -r -d '' node_msg <<- EOM
-	Choose a distributed node type to join to an existing grid. See $DOC_BASE_URL/architecture.html for details.
+	Choose a distributed node type to join to an existing grid. See $DOC_BASE_URL/architecture for details.
 
 	Note: Heavy nodes (HEAVYNODE) are NOT recommended for most users.
 	EOM
@@ -1420,7 +1420,7 @@ whiptail_storage_requirements() {
 
 		You need ${needed_val} to meet minimum requirements.
 
-		Visit $DOC_BASE_URL/hardware.html for more information.
+		Visit $DOC_BASE_URL/hardware for more information.
 
 		Select YES to continue anyway, or select NO to cancel.
 	EOM