securityonion/setup/so-verify

#!/bin/bash

# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.

cd "$(dirname "$0")" || exit 255

source ../salt/common/tools/sbin/so-common
source ./so-functions
source ./so-variables

setup_type=$1

using_iso() {
    if [ "$setup_type" == "iso" ]; then
        return 0
    fi
    return 1
}

# Check entire setup log for errors or unexpected salt states
log_has_errors() {
    # Ignore salt mast cached public key and minion failed to auth because this is a test 
    # to see if the salt key had already been accepted.

    # Ignore failed to connect to ::1 since we have most curls wrapped in a retry.

    # Ignore perl-Error- since that is the name of a Perl package SO installs.

    # Ignore Failed: 0 since that is the salt state output, and we detect state failures
    # via Result: False already.
    
    grep -E "FAILED|Failed|failed|ERROR|Result: False" "$setup_log" | \
        grep -vE "The Salt Master has cached the public key for this node" | \
        grep -vE "Minion failed to authenticate with the master" | \
        grep -vE "Failed to connect to ::1" | \
        grep -vE "Failed to set locale" | \
        grep -vE "perl-Error-" | \
        grep -vE "Failed:\s*?[0-9]+" | \
        grep -vE "Status .* was not found" | \
        grep -vE "Uncaught exception, closing connection" | \
        grep -vE "Exception in callback None" | \
        grep -vE "deprecation: ERROR" | \
        grep -vE "code: 100" | \
        grep -vE "/nsm/rules/sigma*" | \
        grep -vE "/nsm/rules/yara*" | \
        grep -vE "Running scope as unit" &> "$error_log"
    
    if [[ $? -eq 0 ]]; then
        return 0
    fi
    return 1
}

# For ISO installs, we know nothing else can be running on this server, so there should be
# nothing in any mail spool dir.
cron_error_in_mail_spool() {
    count=$(find /var/spool/mail/ -type f -size +0 | wc -l)
    if [[ $count -ne 0 ]]; then
        return 0
    fi
    return 1
}

# so-setup must return a 0 exit code, indicating all containers are up and healthy. Will retry for a limited
# time before giving up.
status_failed() {
    max_retries=120
    wait_secs=10
    retry_attempts=0
    while ! so-status -q; do
        if [[ $retry_attempts -eq $max_retries ]]; then
            return 0
        fi
        retry_attempts=$((retry_attempts+1))
        echo "INFO: so-status returned non-zero exit code; will retry in $wait_secs seconds ($retry_attempts/$max_retries)"
        sleep $wait_secs
    done
    return 1
}

# Creates scripts that are going to be invoked by the automated test system. These scripts will 
# overwritten by Salt, once the minion is accepted into the grid and the salt states run. 
create_temp_scripts_if_missing() {
    if [ ! -f /usr/sbin/so-status ]; then
        if [ -f /root/success ]; then
            echo "echo 'so-status not yet available'" > /usr/sbin/so-status
            echo "exit 0" >> /usr/sbin/so-status

            # Pretend so-test succeeded in the interim period while waiting for this minion to be accepted. This avoids tests getting hung up.
            echo "echo 'so-test not yet available'" > /usr/sbin/so-test
            echo "exit 0" >> /usr/sbin/so-test
            chmod a+x /usr/sbin/so-test
        else
            echo "echo 'Installation failed - so-status not available'" > /usr/sbin/so-status
            echo "exit 100" >> /usr/sbin/so-status
        fi
        chmod a+x /usr/sbin/so-status
    fi
}

main() {
    exit_code=0
    if [ -f /root/success ]; then
        echo "Successfully completed setup a while ago"
    elif [  -f /root/failure ]; then
        echo "WARNING: Failed setup a while ago"
        exit_code=1
    elif log_has_errors; then 
        echo "WARNING: Errors detected during setup"
        exit_code=1
        touch /root/failure
    elif using_iso && cron_error_in_mail_spool; then
        echo "WARNING: Unexpected cron job output in mail spool"
        exit_code=1
        touch /root/failure
    elif is_manager_node && status_failed; then
        echo "WARNING: Containers are not in a healthy state"
        exit_code=1
        touch /root/failure
    else
        echo "Successfully completed setup!"
        touch /root/success
    fi

    create_temp_scripts_if_missing
    
    exit $exit_code
}

main